本文整理汇总了Python中nn_utils.softmax函数的典型用法代码示例。如果您正苦于以下问题：Python softmax函数的具体用法？Python softmax怎么用？Python softmax使用的例子？那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了softmax函数的13个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: answer_step
def answer_step(prev_a, prev_y):
a = self.GRU_update(prev_a, T.concatenate([prev_y, self.q_q]),
self.W_ans_res_in, self.W_ans_res_hid, self.b_ans_res,
self.W_ans_upd_in, self.W_ans_upd_hid, self.b_ans_upd,
self.W_ans_hid_in, self.W_ans_hid_hid, self.b_ans_hid)
y = nn_utils.softmax(T.dot(self.W_a, a))
return [a, y]示例2: new_episode
def new_episode(self, mem):
g, g_updates = theano.scan(fn=self.new_attention_step,
sequences=self.inp_c,
non_sequences=[mem, self.q_q],
outputs_info=T.zeros_like(self.inp_c[0][0]))
if (self.normalize_attention):
g = nn_utils.softmax(g)
e, e_updates = theano.scan(fn=self.new_episode_step,
sequences=[self.inp_c, g],
outputs_info=T.zeros_like(self.inp_c[0]))
return e[-1]示例3: new_img_episode
def new_img_episode(self, mem):
g, g_updates = theano.scan(fn=self.new_img_attention_step,
sequences=self.img_inp_c,
non_sequences=[mem, self.q_q],
outputs_info=T.zeros_like(self.img_inp_c[0][0]))
if (self.normalize_attention):
g = nn_utils.softmax(g)
e, e_updates = theano.scan(fn=self.new_img_episode_step,
sequences=[self.img_inp_c, g],
outputs_info=T.zeros_like(self.img_inp_c[0]))
e_list = []
for index in range(self.batch_size):
e_list.append(e[self.img_seq_len - 1, :, index])
return T.stack(e_list).dimshuffle((1, 0))示例4: new_episode
def new_episode(self, mem):
z, z_updates = theano.scan(fn=self.episode_compute_z,
sequences=self.inp_c,
non_sequences=[mem, self.q_q],
outputs_info=T.zeros_like(self.b_2))
g, g_updates = theano.scan(fn=self.episode_compute_g,
sequences=z,
non_sequences=z,)
if (self.normalize_attention):
g = nn_utils.softmax(g)
self.attentions.append(g)
e, e_updates = theano.scan(fn=self.episode_attend,
sequences=[self.inp_c, g],
outputs_info=T.zeros_like(self.inp_c[0]))
return e[-1] 示例5: __init__
def __init__(self, babi_train_raw, babi_test_raw, word2vec, word_vector_size,
dim, mode, answer_module, input_mask_mode, memory_hops, l2,
normalize_attention, **kwargs):
print "==> not used params in DMN class:", kwargs.keys()
self.vocab = {}
self.ivocab = {}
self.word2vec = word2vec
self.word_vector_size = word_vector_size
self.dim = dim
self.mode = mode
self.answer_module = answer_module
self.input_mask_mode = input_mask_mode
self.memory_hops = memory_hops
self.l2 = l2
self.normalize_attention = normalize_attention
self.train_input, self.train_q, self.train_answer, self.train_input_mask = self._process_input(babi_train_raw)
self.test_input, self.test_q, self.test_answer, self.test_input_mask = self._process_input(babi_test_raw)
self.vocab_size = len(self.vocab)
self.input_var = T.matrix('input_var')
self.q_var = T.matrix('question_var')
self.answer_var = T.iscalar('answer_var')
self.input_mask_var = T.ivector('input_mask_var')
print "==> building input module"
self.W_inp_res_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.word_vector_size))
self.W_inp_res_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_inp_res = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_inp_upd_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.word_vector_size))
self.W_inp_upd_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_inp_upd = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_inp_hid_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.word_vector_size))
self.W_inp_hid_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_inp_hid = nn_utils.constant_param(value=0.0, shape=(self.dim,))
inp_c_history, _ = theano.scan(fn=self.input_gru_step,
sequences=self.input_var,
outputs_info=T.zeros_like(self.b_inp_hid))
self.inp_c = inp_c_history.take(self.input_mask_var, axis=0)
self.q_q, _ = theano.scan(fn=self.input_gru_step,
sequences=self.q_var,
outputs_info=T.zeros_like(self.b_inp_hid))
self.q_q = self.q_q[-1]
print "==> creating parameters for memory module"
self.W_mem_res_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.W_mem_res_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_mem_res = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_mem_upd_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.W_mem_upd_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_mem_upd = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_mem_hid_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.W_mem_hid_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_mem_hid = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_b = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.W_1 = nn_utils.normal_param(std=0.1, shape=(self.dim, 7 * self.dim + 2))
self.W_2 = nn_utils.normal_param(std=0.1, shape=(1, self.dim))
self.b_1 = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.b_2 = nn_utils.constant_param(value=0.0, shape=(1,))
print "==> building episodic memory module (fixed number of steps: %d)" % self.memory_hops
memory = [self.q_q.copy()]
for iter in range(1, self.memory_hops + 1):
current_episode = self.new_episode(memory[iter - 1])
memory.append(self.GRU_update(memory[iter - 1], current_episode,
self.W_mem_res_in, self.W_mem_res_hid, self.b_mem_res,
self.W_mem_upd_in, self.W_mem_upd_hid, self.b_mem_upd,
self.W_mem_hid_in, self.W_mem_hid_hid, self.b_mem_hid))
last_mem = memory[-1]
print "==> building answer module"
self.W_a = nn_utils.normal_param(std=0.1, shape=(self.vocab_size, self.dim))
if self.answer_module == 'feedforward':
self.prediction = nn_utils.softmax(T.dot(self.W_a, last_mem))
elif self.answer_module == 'recurrent':
self.W_ans_res_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim + self.vocab_size))
self.W_ans_res_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_ans_res = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_ans_upd_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim + self.vocab_size))
self.W_ans_upd_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_ans_upd = nn_utils.constant_param(value=0.0, shape=(self.dim,))
#.........这里部分代码省略.........
示例6: __init__
#.........这里部分代码省略.........
img_memory = [self.q_q.copy()]
for iter in range(1, self.memory_hops + 1):
current_episode = self.new_img_episode(img_memory[iter - 1])
img_memory.append(self.GRU_update(img_memory[iter - 1], current_episode,
self.W_img_mem_res_in, self.W_img_mem_res_hid, self.b_img_mem_res,
self.W_img_mem_upd_in, self.W_img_mem_upd_hid, self.b_img_mem_upd,
self.W_img_mem_hid_in, self.W_img_mem_hid_hid, self.b_img_mem_hid))
last_img_mem_raw = img_memory[-1].dimshuffle((1, 0))
#######################################################################
### Concatenating The 2 Memory Modules Representations Assuming the representation as self.batch_size x self.dim ###
combined_mem_raw=T.concatenate([last_mem_raw,last_img_mem_raw],axis=1)
#net = layers.InputLayer(shape=(self.batch_size, self.dim), input_var=last_mem_raw)
net = layers.InputLayer(shape=(self.batch_size, self.dim+self.dim), input_var=combined_mem_raw)
if self.batch_norm:
net = layers.BatchNormLayer(incoming=net)
if self.dropout > 0 and self.mode == 'train':
net = layers.DropoutLayer(net, p=self.dropout)
last_mem = layers.get_output(net).dimshuffle((1, 0))
print "==> building answer module"
#self.W_a = nn_utils.normal_param(std=0.1, shape=(self.vocab_size, self.dim))
self.W_a = nn_utils.normal_param(std=0.1, shape=(self.vocab_size, self.dim+self.dim))
if self.answer_module == 'feedforward':
self.prediction = nn_utils.softmax(T.dot(self.W_a, last_mem))
elif self.answer_module == 'recurrent':
self.W_ans_res_in = nn_utils.normal_param(std=0.1, shape=(2*self.dim, self.dim + self.vocab_size))
self.W_ans_res_hid = nn_utils.normal_param(std=0.1, shape=(2*self.dim, 2*self.dim))
self.b_ans_res = nn_utils.constant_param(value=0.0, shape=(2*self.dim,))
self.W_ans_upd_in = nn_utils.normal_param(std=0.1, shape=(2*self.dim, self.dim + self.vocab_size))
self.W_ans_upd_hid = nn_utils.normal_param(std=0.1, shape=(2*self.dim,2*self.dim))
self.b_ans_upd = nn_utils.constant_param(value=0.0, shape=(2*self.dim,))
self.W_ans_hid_in = nn_utils.normal_param(std=0.1, shape=(2*self.dim, self.dim + self.vocab_size))
self.W_ans_hid_hid = nn_utils.normal_param(std=0.1, shape=(2*self.dim, 2*self.dim))
self.b_ans_hid = nn_utils.constant_param(value=0.0, shape=(2*self.dim,))
def answer_step(prev_a, prev_y):
a = self.GRU_update(prev_a, T.concatenate([prev_y, self.q_q]),
self.W_ans_res_in, self.W_ans_res_hid, self.b_ans_res,
self.W_ans_upd_in, self.W_ans_upd_hid, self.b_ans_upd,
self.W_ans_hid_in, self.W_ans_hid_hid, self.b_ans_hid)
y = nn_utils.softmax(T.dot(self.W_a, a))
return [a, y]
# TODO: add conditional ending
dummy = theano.shared(np.zeros((self.vocab_size, self.batch_size), dtype=floatX))
results, updates = theano.scan(fn=answer_step,
outputs_info=[last_mem, T.zeros_like(dummy)], #(last_mem, y)
n_steps=1)
self.prediction = results[1][-1]
else:
raise Exception("invalid answer_module")示例7: __init__
#.........这里部分代码省略.........
self.W_mem_hid_in = nn_utils.normal_param(std=0.1, shape=(self.memory_hops, self.dim, self.dim))
self.W_mem_hid_hid = nn_utils.normal_param(std=0.1, shape=(self.memory_hops, self.dim, self.dim))
self.b_mem_hid = nn_utils.constant_param(value=0.0, shape=(self.memory_hops, self.dim,))
#self.W_b = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
#self.W_1 = nn_utils.normal_param(std=0.1, shape=(self.dim, 7 * self.dim + 0))
self.W_1 = nn_utils.normal_param(std=0.1, shape=(self.memory_hops, self.dim, 4 * self.dim + 0))
self.W_2 = nn_utils.normal_param(std=0.1, shape=(self.memory_hops, 1, self.dim))
self.b_1 = nn_utils.constant_param(value=0.0, shape=(self.memory_hops, self.dim,))
self.b_2 = nn_utils.constant_param(value=0.0, shape=(self.memory_hops, 1,))
print "==> building episodic memory module (fixed number of steps: %d)" % self.memory_hops
memory = [self.q_q.copy()]
for iter in range(1, self.memory_hops + 1):
self.mem_weight_num = int(iter - 1)
current_episode = self.new_episode(memory[iter - 1])
memory.append(self.GRU_update(memory[iter - 1], current_episode,
self.W_mem_res_in[self.mem_weight_num], self.W_mem_res_hid[self.mem_weight_num], self.b_mem_res[self.mem_weight_num],
self.W_mem_upd_in[self.mem_weight_num], self.W_mem_upd_hid[self.mem_weight_num], self.b_mem_upd[self.mem_weight_num],
self.W_mem_hid_in[self.mem_weight_num], self.W_mem_hid_hid[self.mem_weight_num], self.b_mem_hid[self.mem_weight_num]))
last_mem_raw = memory[-1].dimshuffle(('x', 0))
net = layers.InputLayer(shape=(1, self.dim), input_var=last_mem_raw)
if self.dropout > 0 and self.mode == 'train':
net = layers.DropoutLayer(net, p=self.dropout)
last_mem = layers.get_output(net)[0]
print "==> building answer module"
self.W_a = nn_utils.normal_param(std=0.1, shape=(self.vocab_size, self.dim))
if self.answer_module == 'feedforward':
self.prediction = nn_utils.softmax(T.dot(self.W_a, last_mem))
elif self.answer_module == 'recurrent':
self.W_ans_res_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim + self.vocab_size))
self.W_ans_res_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_ans_res = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_ans_upd_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim + self.vocab_size))
self.W_ans_upd_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_ans_upd = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_ans_hid_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim + self.vocab_size))
self.W_ans_hid_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_ans_hid = nn_utils.constant_param(value=0.0, shape=(self.dim,))
def answer_step(prev_a, prev_y):
a = self.GRU_update(prev_a, T.concatenate([prev_y, self.q_q]),
self.W_ans_res_in, self.W_ans_res_hid, self.b_ans_res,
self.W_ans_upd_in, self.W_ans_upd_hid, self.b_ans_upd,
self.W_ans_hid_in, self.W_ans_hid_hid, self.b_ans_hid)
y = nn_utils.softmax(T.dot(self.W_a, a))
return [a, y]
# add conditional ending?
dummy = theano.shared(np.zeros((self.vocab_size, ), dtype=floatX))
results, updates = theano.scan(fn=answer_step,
outputs_info=[last_mem, T.zeros_like(dummy)],
n_steps=1)
self.prediction = results[1][-1]
else:示例8: __init__
#.........这里部分代码省略.........
self.q_q, _ = theano.scan(fn=self.input_gru_step,
sequences=q_mat,
outputs_info=T.zeros_like(self.b_inp_hid))
self.q_q = self.q_q[-1]
print "==> creating parameters for memory module"
self.W_mem_res_in = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.W_mem_res_hid = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.b_mem_res = theano.shared(lasagne.init.Constant(0.0).sample((self.dim,)), borrow=True)
self.W_mem_upd_in = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.W_mem_upd_hid = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.b_mem_upd = theano.shared(lasagne.init.Constant(0.0).sample((self.dim,)), borrow=True)
self.W_mem_hid_in = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.W_mem_hid_hid = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.b_mem_hid = theano.shared(lasagne.init.Constant(0.0).sample((self.dim,)), borrow=True)
self.W_b = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.W_1 = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, 7 * self.dim)), borrow=True)
self.W_2 = theano.shared(lasagne.init.Normal(0.1).sample((1, self.dim)), borrow=True)
self.b_1 = theano.shared(lasagne.init.Constant(0.0).sample((self.dim,)), borrow=True)
self.b_2 = theano.shared(lasagne.init.Constant(0.0).sample((1,)), borrow=True)
print "==> building episodic memory module (fixed number of steps: %d)" % self.memory_hops
memory = [self.q_q.copy()] # (dim, 1)
for iter in range(1, self.memory_hops + 1):
current_episode = self.new_episode(memory[iter - 1])
memory.append(self.GRU_update(memory[iter - 1], current_episode,
self.W_mem_res_in, self.W_mem_res_hid, self.b_mem_res,
self.W_mem_upd_in, self.W_mem_upd_hid, self.b_mem_upd,
self.W_mem_hid_in, self.W_mem_hid_hid, self.b_mem_hid))
last_mem_raw = memory[-1].dimshuffle('x', 0) # (batch_size=1, dim)
net = layers.InputLayer(shape=(1, self.dim), input_var=last_mem_raw)
if self.dropout > 0 and self.mode == 'train':
net = layers.DropoutLayer(net, p=self.dropout)
last_mem = layers.get_output(net)[0]
print "==> building options module"
self.c_vecs = []
for choice in [ca_mat, cb_mat, cc_mat, cd_mat]:
history, _ = theano.scan(fn=self.input_gru_step,
sequences=choice,
outputs_info=T.zeros_like(self.b_inp_hid))
self.c_vecs.append(history[-1])
self.c_vecs = T.stack(self.c_vecs).transpose((1, 0)) # (dim, 4)
print "==> building answer module"
self.W_a = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.prediction = nn_utils.softmax(T.dot(T.dot(self.W_a, last_mem),self.c_vecs))
print "==> collecting all parameters" # embedding matrix is not trained
self.params = [self.embed,
self.W_inp_res_in, self.W_inp_res_hid, self.b_inp_res,
self.W_inp_upd_in, self.W_inp_upd_hid, self.b_inp_upd,
self.W_inp_hid_in, self.W_inp_hid_hid, self.b_inp_hid,
self.W_mem_res_in, self.W_mem_res_hid, self.b_mem_res,
self.W_mem_upd_in, self.W_mem_upd_hid, self.b_mem_upd,
self.W_mem_hid_in, self.W_mem_hid_hid, self.b_mem_hid,
self.W_b, self.W_1, self.W_2, self.b_1, self.b_2, self.W_a]
print "==> building loss layer and computing updates"
self.loss_ce = T.nnet.categorical_crossentropy(self.prediction.dimshuffle('x', 0), T.stack([self.ans_var]))[0]
if self.l2 > 0:
self.loss_l2 = self.l2 * nn_utils.l2_reg(self.params)
else:
self.loss_l2 = 0
self.loss = self.loss_ce + self.loss_l2
updates = lasagne.updates.adadelta(self.loss, self.params)
if self.mode == 'train':
print "==> compiling train_fn"
self.train_fn = theano.function(inputs=[self.inp_var, self.q_var, self.ans_var,
self.ca_var, self.cb_var, self.cc_var, self.cd_var,
self.input_mask_var],
allow_input_downcast = True,
outputs=[self.prediction, self.loss],
updates=updates)
self.attentions = T.stack(self.attentions)
print "==> compiling test_fn"
self.test_fn = theano.function(inputs=[self.inp_var, self.q_var, self.ans_var,
self.ca_var, self.cb_var, self.cc_var, self.cd_var,
self.input_mask_var],
allow_input_downcast = True,
outputs=[self.prediction, self.loss, self.attentions, self.inp_c, self.q_q, last_mem])
if self.mode == 'train':
print "==> computing gradients (for debugging)"
gradient = T.grad(self.loss, self.params)
self.get_gradient_fn = theano.function(inputs=[self.inp_var, self.q_var, self.ans_var,
self.ca_var, self.cb_var, self.cc_var, self.cd_var,
self.input_mask_var],
allow_input_downcast = True,
outputs=gradient)示例9: __init__
def __init__(self, train_raw, dev_raw, test_raw, word2vec, word_vector_size,
dim, mode, input_mask_mode, memory_hops, l2, normalize_attention, dropout, **kwargs):
print "generate one-word answer for mctest"
print "==> not used params in DMN class:", kwargs.keys()
self.word2vec = word2vec
self.word_vector_size = word_vector_size
self.vocab_size = len(word2vec)
self.dim = dim # hidden state size
self.mode = mode
self.input_mask_mode = input_mask_mode
self.memory_hops = memory_hops
self.l2 = l2
self.normalize_attention = normalize_attention
self.dropout = dropout
self.train_input, self.train_q, self.train_answer, self.train_input_mask = self._process_input(train_raw)
self.dev_input, self.dev_q, self.dev_answer, self.dev_input_mask = self._process_input(dev_raw)
self.test_input, self.test_q, self.test_answer, self.test_input_mask = self._process_input(test_raw)
self.input_var = T.matrix('input_var')
self.q_var = T.matrix('question_var')
self.answer_var = T.iscalar('answer_var')
self.input_mask_var = T.ivector('input_mask_var')
self.attentions = []
print "==> building input module"
self.W_inp_res_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.word_vector_size))
self.W_inp_res_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_inp_res = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_inp_upd_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.word_vector_size))
self.W_inp_upd_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_inp_upd = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_inp_hid_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.word_vector_size))
self.W_inp_hid_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_inp_hid = nn_utils.constant_param(value=0.0, shape=(self.dim,))
inp_c_history, _ = theano.scan(fn=self.input_gru_step,
sequences=self.input_var,
outputs_info=T.zeros_like(self.b_inp_hid))
self.inp_c = inp_c_history.take(self.input_mask_var, axis=0)
self.q_q, _ = theano.scan(fn=self.input_gru_step,
sequences=self.q_var,
outputs_info=T.zeros_like(self.b_inp_hid))
self.q_q = self.q_q[-1]
print "==> creating parameters for memory module"
self.W_mem_res_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.W_mem_res_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_mem_res = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_mem_upd_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.W_mem_upd_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_mem_upd = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_mem_hid_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.W_mem_hid_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_mem_hid = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_b = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.W_1 = nn_utils.normal_param(std=0.1, shape=(self.dim, 7 * self.dim + 2))
self.W_2 = nn_utils.normal_param(std=0.1, shape=(1, self.dim))
self.b_1 = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.b_2 = nn_utils.constant_param(value=0.0, shape=(1,))
print "==> building episodic memory module (fixed number of steps: %d)" % self.memory_hops
memory = [self.q_q.copy()]
for iter in range(1, self.memory_hops + 1):
current_episode = self.new_episode(memory[iter - 1])
memory.append(self.GRU_update(memory[iter - 1], current_episode,
self.W_mem_res_in, self.W_mem_res_hid, self.b_mem_res,
self.W_mem_upd_in, self.W_mem_upd_hid, self.b_mem_upd,
self.W_mem_hid_in, self.W_mem_hid_hid, self.b_mem_hid))
last_mem_raw = memory[-1].dimshuffle(('x', 0))
net = layers.InputLayer(shape=(1, self.dim), input_var=last_mem_raw)
if self.dropout > 0 and self.mode == 'train':
net = layers.DropoutLayer(net, p=self.dropout)
last_mem = layers.get_output(net)[0]
self.attentions = T.stack(self.attentions)
print "==> building answer module"
self.W_a = nn_utils.normal_param(std=0.1, shape=(self.vocab_size, self.dim))
self.prediction = nn_utils.softmax(T.dot(self.W_a, last_mem))
print "==> collecting all parameters"
self.params = [self.W_inp_res_in, self.W_inp_res_hid, self.b_inp_res,
self.W_inp_upd_in, self.W_inp_upd_hid, self.b_inp_upd,
self.W_inp_hid_in, self.W_inp_hid_hid, self.b_inp_hid,
self.W_mem_res_in, self.W_mem_res_hid, self.b_mem_res,
self.W_mem_upd_in, self.W_mem_upd_hid, self.b_mem_upd,
#.........这里部分代码省略.........
示例10: __init__
#.........这里部分代码省略.........
self.W_mem_hid_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_mem_hid = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_b = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.W_1 = nn_utils.normal_param(std=0.1, shape=(self.dim, 7 * self.dim + 0))
self.W_2 = nn_utils.normal_param(std=0.1, shape=(1, self.dim))
self.b_1 = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.b_2 = nn_utils.constant_param(value=0.0, shape=(1,))
print "==> building episodic memory module (fixed number of steps: %d)" % self.memory_hops
memory = [self.q_q.copy()]
for iter in range(1, self.memory_hops + 1):
current_episode = self.new_episode(memory[iter - 1])
memory.append(self.GRU_update(memory[iter - 1], current_episode,
self.W_mem_res_in, self.W_mem_res_hid, self.b_mem_res,
self.W_mem_upd_in, self.W_mem_upd_hid, self.b_mem_upd,
self.W_mem_hid_in, self.W_mem_hid_hid, self.b_mem_hid))
last_mem_raw = memory[-1].dimshuffle((1, 0))
net = layers.InputLayer(shape=(self.batch_size, self.dim), input_var=last_mem_raw)
if self.batch_norm:
net = layers.BatchNormLayer(incoming=net)
if self.dropout > 0 and self.mode == 'train':
net = layers.DropoutLayer(net, p=self.dropout)
last_mem = layers.get_output(net).dimshuffle((1, 0))
print "==> building answer module"
self.W_a = nn_utils.normal_param(std=0.1, shape=(self.vocab_size, self.dim))
if self.answer_module == 'feedforward':
self.prediction = nn_utils.softmax(T.dot(self.W_a, last_mem))
elif self.answer_module == 'recurrent':
self.W_ans_res_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim + self.vocab_size))
self.W_ans_res_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_ans_res = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_ans_upd_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim + self.vocab_size))
self.W_ans_upd_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_ans_upd = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_ans_hid_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim + self.vocab_size))
self.W_ans_hid_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_ans_hid = nn_utils.constant_param(value=0.0, shape=(self.dim,))
def answer_step(prev_a, prev_y):
a = self.GRU_update(prev_a, T.concatenate([prev_y, self.q_q]),
self.W_ans_res_in, self.W_ans_res_hid, self.b_ans_res,
self.W_ans_upd_in, self.W_ans_upd_hid, self.b_ans_upd,
self.W_ans_hid_in, self.W_ans_hid_hid, self.b_ans_hid)
y = nn_utils.softmax(T.dot(self.W_a, a))
return [a, y]
# TODO: add conditional ending
dummy = theano.shared(np.zeros((self.vocab_size, self.batch_size), dtype=floatX))
results, updates = theano.scan(fn=self.answer_step,
outputs_info=[last_mem, T.zeros_like(dummy)], #(last_mem, y)
n_steps=1)
self.prediction = results[1][-1]
else:
raise Exception("invalid answer_module")示例11: __init__
#.........这里部分代码省略.........
self.W_mem_upd_in = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.W_mem_upd_hid = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.b_mem_upd = theano.shared(lasagne.init.Constant(0.0).sample((self.dim,)), borrow=True)
self.W_mem_hid_in = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.W_mem_hid_hid = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.b_mem_hid = theano.shared(lasagne.init.Constant(0.0).sample((self.dim,)), borrow=True)
self.W_b = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.W_1 = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, 7 * self.dim + 2)), borrow=True)
self.W_2 = theano.shared(lasagne.init.Normal(0.1).sample((1, self.dim)), borrow=True)
self.b_1 = theano.shared(lasagne.init.Constant(0.0).sample((self.dim,)), borrow=True)
self.b_2 = theano.shared(lasagne.init.Constant(0.0).sample((1,)), borrow=True)
print "==> building episodic memory module (fixed number of steps: %d)" % self.memory_hops
memory = [self.q_q.copy()]
for iter in range(1, self.memory_hops + 1):
current_episode = self.new_episode(memory[iter - 1])
memory.append(self.GRU_update(memory[iter - 1], current_episode,
self.W_mem_res_in, self.W_mem_res_hid, self.b_mem_res,
self.W_mem_upd_in, self.W_mem_upd_hid, self.b_mem_upd,
self.W_mem_hid_in, self.W_mem_hid_hid, self.b_mem_hid))
last_mem = memory[-1]
print "==> building answer module"
self.W_a = theano.shared(lasagne.init.Normal(0.1).sample((self.vocab_size, self.dim)), borrow=True)
if self.answer_module == 'feedforward':
self.prediction = nn_utils.softmax(T.dot(self.W_a, last_mem))
elif self.answer_module == 'recurrent':
self.W_ans_res_in = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim + self.vocab_size)), borrow=True)
self.W_ans_res_hid = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.b_ans_res = theano.shared(lasagne.init.Constant(0.0).sample((self.dim,)), borrow=True)
self.W_ans_upd_in = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim + self.vocab_size)), borrow=True)
self.W_ans_upd_hid = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.b_ans_upd = theano.shared(lasagne.init.Constant(0.0).sample((self.dim,)), borrow=True)
self.W_ans_hid_in = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim + self.vocab_size)), borrow=True)
self.W_ans_hid_hid = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.b_ans_hid = theano.shared(lasagne.init.Constant(0.0).sample((self.dim,)), borrow=True)
def answer_step(prev_a, prev_y):
a = self.GRU_update(prev_a, T.concatenate([prev_y, self.q_q]),
self.W_ans_res_in, self.W_ans_res_hid, self.b_ans_res,
self.W_ans_upd_in, self.W_ans_upd_hid, self.b_ans_upd,
self.W_ans_hid_in, self.W_ans_hid_hid, self.b_ans_hid)
y = nn_utils.softmax(T.dot(self.W_a, a))
return [a, y]
# TODO: add conditional ending
dummy = theano.shared(np.zeros((self.vocab_size, self.batch_size), dtype=floatX))
results, updates = theano.scan(fn=self.answer_step,
outputs_info=[last_mem, T.zeros_like(dummy)], #(last_mem, y)
n_steps=1)
self.prediction = results[1][-1]
else:
raise Exception("invalid answer_module")示例12: __init__
#.........这里部分代码省略.........
inp_c_history, _ = theano.scan(fn=self.input_gru_step,
sequences=self.input_var,
outputs_info=T.zeros_like(self.b_inp_hid))
self.end_reading = nn_utils.constant_param(value=0.0,shape=(1,self.dim))
inp_c_tag = T.concatenate([inp_c_history,self.end_reading],axis=0)
self.inp_c = inp_c_tag.take(self.input_mask_var, axis=0) #(facts_len,dim)
self.q_q, _ = theano.scan(fn=self.input_gru_step,
sequences=self.q_var,
outputs_info=T.zeros_like(self.b_inp_hid))
self.q_q = self.q_q[-1] #(1,dim)
print "==> creating parameters for memory module"
self.W_mem_res_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.W_mem_res_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_mem_res = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_mem_upd_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.W_mem_upd_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_mem_upd = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_mem_hid_in = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.W_mem_hid_hid = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.b_mem_hid = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.W_b = nn_utils.normal_param(std=0.1, shape=(self.dim, self.dim))
self.W_1 = nn_utils.normal_param(std=0.1, shape=(self.dim, 7 * self.dim + 2))
self.W_2 = nn_utils.normal_param(std=0.1, shape=(1, self.dim))
self.b_1 = nn_utils.constant_param(value=0.0, shape=(self.dim,))
self.b_2 = nn_utils.constant_param(value=0.0, shape=(1,))
print "==> building episodic memory module (fixed number of steps: %d)" % self.memory_hops
memory = [self.q_q.copy()]
for iter in range(0, self.memory_hops):
current_episode, g = self.new_episode(memory[iter])
self.attentions.append(g)
memory.append(self.GRU_update(memory[iter], current_episode,
self.W_mem_res_in, self.W_mem_res_hid, self.b_mem_res,
self.W_mem_upd_in, self.W_mem_upd_hid, self.b_mem_upd,
self.W_mem_hid_in, self.W_mem_hid_hid, self.b_mem_hid))
last_mem_raw = memory[-1].dimshuffle(('x', 0))
net = layers.InputLayer(shape=(1, self.dim), input_var=last_mem_raw)
if self.dropout > 0 and self.mode == 'train':
net = layers.DropoutLayer(net, p=self.dropout)
last_mem = layers.get_output(net)[0]
self.attentions = T.stack(self.attentions) #(memory_hops, fact_cnt)
print "==> building answer module"
self.W_a = nn_utils.normal_param(std=0.1, shape=(self.vocab_size, self.dim))
self.prediction = nn_utils.softmax(T.dot(self.W_a, last_mem))
print "==> collecting all parameters"
self.params = [self.W_inp_res_in, self.W_inp_res_hid, self.b_inp_res,
self.W_inp_upd_in, self.W_inp_upd_hid, self.b_inp_upd,
self.W_inp_hid_in, self.W_inp_hid_hid, self.b_inp_hid,
self.W_mem_res_in, self.W_mem_res_hid, self.b_mem_res,
self.W_mem_upd_in, self.W_mem_upd_hid, self.b_mem_upd,
self.W_mem_hid_in, self.W_mem_hid_hid, self.b_mem_hid, self.W_b,
self.W_1, self.W_2, self.b_1, self.b_2, self.W_a]
print "==> building loss layer and computing updates"
self.loss_ce = T.nnet.categorical_crossentropy(self.prediction.dimshuffle('x', 0),
T.stack([self.answer_var]))[0]
if self.l2 > 0:
self.loss_l2 = self.l2 * nn_utils.l2_reg(self.params)
else:
self.loss_l2 = 0
self.loss_gate = T.nnet.categorical_crossentropy(self.attentions, self.gates_var).mean()
self.loss = self.loss_ce + self.loss_l2 + self.loss_gate
updates = lasagne.updates.adam(self.loss, self.params)
#updates = lasagne.updates.momentum(self.loss, self.params, learning_rate=0.0003)
if self.mode == 'train':
print "==> compiling train_fn"
self.train_fn = theano.function(inputs=[self.input_var, self.q_var, self.answer_var, self.input_mask_var, self.gates_var],
allow_input_downcast = True,
outputs=[self.prediction, self.loss, self.attentions],
updates=updates)
print "==> compiling test_fn"
self.test_fn = theano.function(inputs=[self.input_var, self.q_var, self.answer_var, self.input_mask_var, self.gates_var],
allow_input_downcast = True,
outputs=[self.prediction, self.loss, self.attentions])
if self.mode == 'train':
print "==> computing gradients (for debugging)"
gradient = T.grad(self.loss, self.params)
self.get_gradient_fn = theano.function(inputs=[self.input_var, self.q_var, self.answer_var, self.input_mask_var, self.gates_var],
allow_input_downcast = True, outputs=gradient) 示例13: __init__
#.........这里部分代码省略.........
self.inp_c = inp_c_history.take(self.input_mask_var, axis=0)
self.q_q, _ = theano.scan(fn=self.input_gru_step,
sequences=self.q_var,
outputs_info=T.zeros_like(self.b_inp_hid))
self.q_q = self.q_q[-1]
self.c_vecs = []
for choice in [self.ca_var, self.cb_var, self.cc_var, self.cd_var]:
history, _ = theano.scan(fn=self.input_gru_step,
sequences=choice,
outputs_info=T.zeros_like(self.b_inp_hid))
self.c_vecs.append(history[-1])
self.c_vecs = T.stack(self.c_vecs).transpose((1, 0)) # (dim, 4)
self.inp_c = T.stack([self.inp_c] * 4).transpose((1, 2, 0)) # (fact_cnt, dim, 4)
self.q_q = T.stack([self.q_q] * 4).transpose((1, 0)) # (dim, 4)
print "==> creating parameters for memory module"
self.W_mem_res_in = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.W_mem_res_hid = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.b_mem_res = theano.shared(lasagne.init.Constant(0.0).sample((self.dim,)), borrow=True)
self.W_mem_upd_in = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.W_mem_upd_hid = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.b_mem_upd = theano.shared(lasagne.init.Constant(0.0).sample((self.dim,)), borrow=True)
self.W_mem_hid_in = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.W_mem_hid_hid = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.b_mem_hid = theano.shared(lasagne.init.Constant(0.0).sample((self.dim,)), borrow=True)
self.W_b = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, self.dim)), borrow=True)
self.W_1 = theano.shared(lasagne.init.Normal(0.1).sample((self.dim, 10 * self.dim + 3)), borrow=True)
self.W_2 = theano.shared(lasagne.init.Normal(0.1).sample((1, self.dim)), borrow=True)
self.b_1 = theano.shared(lasagne.init.Constant(0.0).sample((self.dim,)), borrow=True)
self.b_2 = theano.shared(lasagne.init.Constant(0.0).sample((1,)), borrow=True)
print "==> building episodic memory module (fixed number of steps: %d)" % self.memory_hops
memory = [self.q_q.copy()] # (dim, 4)
for iter in range(1, self.memory_hops + 1):
current_episode = self.new_episode(memory[iter - 1])
memory.append(self.GRU_update_batch(memory[iter - 1], current_episode,
self.W_mem_res_in, self.W_mem_res_hid, self.b_mem_res,
self.W_mem_upd_in, self.W_mem_upd_hid, self.b_mem_upd,
self.W_mem_hid_in, self.W_mem_hid_hid, self.b_mem_hid))
last_mem = memory[-1].flatten()
print "==> building answer module"
self.W_a = theano.shared(lasagne.init.Normal(0.1).sample((self.vocab_size, 4 * self.dim)), borrow=True)
self.prediction = nn_utils.softmax(T.dot(self.W_a, last_mem))
print "==> collecting all parameters"
self.params = [self.W_inp_res_in, self.W_inp_res_hid, self.b_inp_res,
self.W_inp_upd_in, self.W_inp_upd_hid, self.b_inp_upd,
self.W_inp_hid_in, self.W_inp_hid_hid, self.b_inp_hid,
self.W_mem_res_in, self.W_mem_res_hid, self.b_mem_res,
self.W_mem_upd_in, self.W_mem_upd_hid, self.b_mem_upd,
self.W_mem_hid_in, self.W_mem_hid_hid, self.b_mem_hid,
self.W_b, self.W_1, self.W_2, self.b_1, self.b_2, self.W_a]
print "==> building loss layer and computing updates"
self.loss_ce = T.nnet.categorical_crossentropy(self.prediction.dimshuffle('x', 0), T.stack([self.ans_var]))[0]
if self.l2 > 0:
self.loss_l2 = self.l2 * nn_utils.l2_reg(self.params)
else:
self.loss_l2 = 0
self.loss = self.loss_ce + self.loss_l2
updates = lasagne.updates.adadelta(self.loss, self.params)
if self.mode == 'train':
print "==> compiling train_fn"
self.train_fn = theano.function(inputs=[self.inp_var, self.q_var, self.ans_var,
self.ca_var, self.cb_var, self.cc_var, self.cd_var,
self.input_mask_var],
outputs=[self.prediction, self.loss],
updates=updates)
print "==> compiling test_fn"
self.test_fn = theano.function(inputs=[self.inp_var, self.q_var, self.ans_var,
self.ca_var, self.cb_var, self.cc_var, self.cd_var,
self.input_mask_var],
outputs=[self.prediction, self.loss, self.inp_c, self.q_q, last_mem])
if self.mode == 'train':
print "==> computing gradients (for debugging)"
gradient = T.grad(self.loss, self.params)
self.get_gradient_fn = theano.function(inputs=[self.inp_var, self.q_var, self.ans_var,
self.ca_var, self.cb_var, self.cc_var, self.cd_var,
self.input_mask_var], outputs=gradient)