Python FunctionSet.l1方法代码示例

# 需要导入模块: from chainer import FunctionSet [as 别名]
# 或者: from chainer.FunctionSet import l1 [as 别名]
class CNN3_Model(ModelBase):
    u"""see: http://aidiary.hatenablog.com/entry/20151007/1444223445"""
    def __init__(self, input_size=32):
        super(CNN3_Model, self).__init__()
        # F.Convolution2D(in_channel, out_channel, filter_size)
        self.model = FunctionSet(  # 1*32*32 -(conv)-> 20*28*28 -(pool)-> 20*14*14
                                 conv1=F.Convolution2D(1, 20, 5),
                                   # 20*14*14 -(conv)-> 50*10*10 -(pool)-> 50*5*5=1250
                                 conv2=F.Convolution2D(20, 50, 5),
                                 l1=F.Linear(1250, 300),
                                 l2=F.Linear(300, 2))

    def forward(self, x_data, y_data, train=True):
        u"""return loss, accuracy"""
        x, t = Variable(x_data), Variable(y_data)
        h1 = F.max_pooling_2d(F.relu(self.model.conv1(x)), 2)
        h2 = F.max_pooling_2d(F.relu(self.model.conv2(h1)), 2)
        h3 = F.dropout(F.relu(self.model.l1(h2)), train=train)
        y  = self.model.l2(h3)
        # 多クラス分類なので誤差関数としてソフトマックス関数の
        # 交差エントロピー関数を用いて、誤差を導出。最低でもlossは必要
        return {
                "loss": F.softmax_cross_entropy(y, t),
                "accuracy": F.accuracy(y, t)
                }

# 需要导入模块: from chainer import FunctionSet [as 别名]
# 或者: from chainer.FunctionSet import l1 [as 别名]
class Model1:

    def __init__(self, model):
        if isinstance(model, tuple):
            input_dims, n_units, output_dims = model
            self.model = FunctionSet(l1=F.Linear(input_dims, n_units),
                                     l2=F.Linear(n_units, n_units),
                                     l3=F.Linear(n_units, output_dims))
        else:
            self.model = model

    def __call__(self):
        return self.model

    # Neural net architecture
    # ニューラルネットの構造
    def forward(self, x_data, y_data, train=True):
        x = Variable(x_data)
        if not y_data is None: t = Variable(y_data)
        h1 = F.dropout(F.relu(self.model.l1(x)),  train=train)
        h2 = F.dropout(F.relu(self.model.l2(h1)), train=train)
        y  = self.model.l3(h2)
        if not y_data is None:
            # 多クラス分類なので誤差関数としてソフトマックス関数の
            # 交差エントロピー関数を用いて、誤差を導出
            return F.softmax_cross_entropy(y, t), F.accuracy(y, t), y
        else:
            return y

    def evaluate(self, x_data):
        return self.forward(x_data, None, train=False)

# 需要导入模块: from chainer import FunctionSet [as 别名]
# 或者: from chainer.FunctionSet import l1 [as 别名]
class DeepLearning:
    def __init__(self, input_size, hidden_size, output_size):
        self.model = FunctionSet(l1=F.Linear(input_size, hidden_size),
                    l2=F.Linear(hidden_size, hidden_size),
                    l3=F.Linear(hidden_size, output_size))
        self.optimizer = optimizers.Adam()
        self.optimizer.setup(self.model.collect_parameters())



    def batch(self, X_train, y_train, batch_size, perm):
        train_size = X_train.shape[0]

        for i in xrange(0, train_size, batch_size):
            X_batch = X_train[perm[i: i+batch_size]]
            y_batch = y_train[perm[i: i+batch_size]]

            # Chainer用に型変換
            x = Variable(X_batch)
            t = Variable(y_batch)

            self.optimizer.zero_grads()
            y = self.forward(x)  # 予測結果

            loss = F.softmax_cross_entropy(y, t)
            loss.backward()

            self.optimizer.update()


    def forward(self, x, train=True):
        h1 = F.dropout(F.sigmoid(self.model.l1(x)),  train=train)
        h2 = F.dropout(F.sigmoid(self.model.l2(h1)), train=train)
        return self.model.l3(h2)


    def predicate(self, x_data):
        x = np.array([x_data], dtype=np.float32)
        x = Variable(x)
        y = self.forward(x, train=False)
        return np.argmax(y.data)


    def save(self, fpath):
        pickle.dump(self.model, open(fpath, 'wb'), -1)


    def load(self, fpath):
        self.model = pickle.load(open(fpath,'rb'))

# 需要导入模块: from chainer import FunctionSet [as 别名]
# 或者: from chainer.FunctionSet import l1 [as 别名]
class LogisticRegressionEstimator(ChainerClassifier):
    def __init__(self, net_hidden=100, net_out=5, **params):
        ChainerClassifier.__init__(self, **params)
        self.net_hidden = net_hidden
        self.net_out    = net_out
        self.param_names.append('net_hidden')
        self.param_names.append('net_out')

    def setup_network(self, n_features):
        self.network = FunctionSet(
            l1 = F.Linear(n_features,      self.net_hidden),
            l2 = F.Linear(self.net_hidden, self.net_out)
        )

    def forward_inner(self, x, train=True):
        h = F.relu(self.network.l1(x))
        y = self.network.l2(h)
        return y

# 需要导入模块: from chainer import FunctionSet [as 别名]
# 或者: from chainer.FunctionSet import l1 [as 别名]
class NN3_Model(ModelBase):
    def __init__(self, input_dim=748, n_units=1000):
        super(NN3_Model, self).__init__()
        self.n_units = n_units
        self.model = FunctionSet(l1=F.Linear(input_dim, n_units),
                                 l2=F.Linear(n_units, n_units),
                                 l3=F.Linear(n_units, 2))

    def forward(self, x_data, y_data, train=True):
        u"""return loss, accuracy"""
        x, t = Variable(x_data), Variable(y_data)
        h1 = F.dropout(F.relu(self.model.l1(x)),  train=train)
        h2 = F.dropout(F.relu(self.model.l2(h1)), train=train)
        y  = self.model.l3(h2)
        # 多クラス分類なので誤差関数としてソフトマックス関数の
        # 交差エントロピー関数を用いて、誤差を導出。最低でもlossは必要
        return {
                "loss": F.softmax_cross_entropy(y, t),
                "accuracy": F.accuracy(y, t)
                }

# 需要导入模块: from chainer import FunctionSet [as 别名]
# 或者: from chainer.FunctionSet import l1 [as 别名]
class CNN_class:
    def __init__(self):
        self.model = FunctionSet(
            l1 = F.Convolution2D(4, 32, ksize=8, stride=4, nobias=False, wscale=np.sqrt(2)),
            l2 = F.Convolution2D(32, 64, ksize=4, stride=2, nobias=False, wscale=np.sqrt(2)),
            l3 = F.Convolution2D(64, 64, ksize=3, stride=1, nobias=False, wscale=np.sqrt(2))
)
        
        self.model.l1.W = np.load('elite/l1_W.npy')
        self.model.l1.b = np.load('elite/l1_b.npy')
        self.model.l2.W = np.load('elite/l2_W.npy')
        self.model.l2.b = np.load('elite/l2_b.npy')
        self.model.l3.W = np.load('elite/l3_W.npy')
        self.model.l3.b = np.load('elite/l3_b.npy')


    def CNN_forward(self, state):
        h1 = F.relu(self.model.l1(state / 254.0))
        h2 = F.relu(self.model.l2(h1))
        h3 = F.relu(self.model.l3(h2))

        return h3

# 需要导入模块: from chainer import FunctionSet [as 别名]
# 或者: from chainer.FunctionSet import l1 [as 别名]
class MLP(object):
	def __init__(
		self,
		data,
		target,
		n_inputs=784,
		n_hidden=784,
		n_outputs=10,
		gpu=-1
	):

		self.model = FunctionSet(
			l1=F.Linear(n_inputs, n_hidden),
			l2=F.Linear(n_hidden, n_hidden),
			l3=F.Linear(n_hidden, n_outputs)
		)

		if gpu >= 0:
			self.model.to_gpu()

		self.x_train, self.x_test = data
		self.y_train, self.y_test = target

		self.n_train = len(self.y_train)
		self.n_test = len(self.y_test)

		self.gpu = gpu
		self.optimizer = optimizers.Adam()
		self.optimizer.setup(self.model)

		self.train_accuracies = []
		self.train_losses = []
		self.test_accuracies = []
		self.test_losses = []

	@property
	def xp(self):
		return cuda.cupy if self.gpu >= 0 else numpy

	def forward(self, x_data, y_data, train=True):
		x, t = Variable(x_data), Variable(y_data)
		h1 = F.dropout(F.relu(self.model.l1(x)), train=train)
		h2 = F.dropout(F.relu(self.model.l2(h1)), train=train)
		y = self.model.l3(h2)
		return F.softmax_cross_entropy(y, t), F.accuracy(y, t)

	def train_and_test(self, n_epoch=20, batchsize=100):
		for epoch in xrange(1, n_epoch + 1):
			logging.info('epoch {}'.format(epoch))

			perm = numpy.random.permutation(self.n_train)
			sum_accuracy = 0
			sum_loss = 0
			for i in xrange(0, self.n_train, batchsize):
				x_batch = self.xp.asarray(self.x_train[perm[i:i+batchsize]])
				y_batch = self.xp.asarray(self.y_train[perm[i:i+batchsize]])

				real_batchsize = len(x_batch)

				self.optimizer.zero_grads()
				loss, acc = self.forward(x_batch, y_batch)
				loss.backward()
				self.optimizer.update()

				sum_loss += float(cuda.to_cpu(loss.data)) * real_batchsize
				sum_accuracy += float(cuda.to_cpu(acc.data)) * real_batchsize

			self.train_accuracies.append(sum_accuracy / self.n_train)
			self.train_losses.append(sum_loss / self.n_train)

			logging.info(
				'train mean loss={}, accuracy={}'.format(
					sum_loss / self.n_train,
					sum_accuracy / self.n_train
				)
			)

			# evalation
			sum_accuracy = 0
			sum_loss = 0
			for i in xrange(0, self.n_test, batchsize):
				x_batch = self.xp.asarray(self.x_test[i:i+batchsize])
				y_batch = self.xp.asarray(self.y_test[i:i+batchsize])

				real_batchsize = len(x_batch)

				loss, acc = self.forward(x_batch, y_batch, train=False)

				sum_loss += float(cuda.to_cpu(loss.data)) * real_batchsize
				sum_accuracy += float(cuda.to_cpu(acc.data)) * real_batchsize

			self.test_accuracies.append(sum_accuracy / self.n_test)
			self.test_accuracies.append(sum_loss / self.n_test)
			logging.info(
				'test mean loss={}, accuracy={}'.format(
					sum_loss / self.n_test,
					sum_accuracy / self.n_test
				)
			)

# 需要导入模块: from chainer import FunctionSet [as 别名]
# 或者: from chainer.FunctionSet import l1 [as 别名]

# output loss
f = open(loss_output, 'w')
for i in range(1, n_epoch + 1):
    strs = '{0:05.0f} {1:.2f} {2:.2f}\n'.format(i, loss_train[i - 1], loss_test[i - 1])
    f.writelines(strs)
f.close()


# test
print('-----')
print('starting to make test data with model')

x = Variable(cuda.to_gpu(x_test[1000:1000 + 200].reshape((200, default_bitrate))))
h1 = F.dropout(F.relu(model.l1(x)),  train=False)
y = F.dropout(model.l2(h1), train=False)
#print(x.data)
#print(y.data)


x_range = np.arange(0, default_bitrate * 200, 1)
print('test  mean loss={}'.format(F.mean_squared_error(y, x).data))
#print(x.data.ndim, x_range.ndim)
#plt.plot(x_range, y.data[0])
#plt.plot(x_range, t.data[0])
#plt.show()
x_datas = []
t_datas = []
y_datas = []
x_datas.extend(x_range)

# 需要导入模块: from chainer import FunctionSet [as 别名]
# 或者: from chainer.FunctionSet import l1 [as 别名]
class DQN_class:
    # Hyper-Parameters
    gamma = 0.99  # Discount factor
    initial_exploration = 10**4  # Initial exploratoin. original: 5x10^4
    replay_size = 32  # Replay (batch) size
    target_model_update_freq = 10**4  # Target update frequancy. original: 10^4
    data_size = 10**5  # Data size of history. original: 10^6

    def __init__(self, enable_controller=[0, 3, 4]):
        self.num_of_actions = len(enable_controller)
        self.enable_controller = enable_controller  # Default setting : "Pong"

        print "Initializing DQN..."
#	Initialization for Chainer 1.1.0 or older.
#        print "CUDA init"
#        cuda.init()

        print "Model Building"
        self.model = FunctionSet(
            l1=F.Convolution2D(4, 16, ksize=8, stride=4, wscale=np.sqrt(2)),
            l2=F.Convolution2D(16, 32, ksize=4, stride=2, wscale=np.sqrt(2)),
            l3=F.Linear(2592, 256),
            q_value=F.Linear(256, self.num_of_actions,
                             initialW=np.zeros((self.num_of_actions, 256),
                                               dtype=np.float32))
        ).to_gpu()

        print "Initizlizing Optimizer"
        self.optimizer = optimizers.RMSpropGraves(lr=0.0002, alpha=0.3, momentum=0.2)
        self.optimizer.setup(self.model.collect_parameters())

        # History Data :  D=[s, a, r, s_dash, end_episode_flag]
        self.D = [np.zeros((self.data_size, 4, 84, 84), dtype=np.uint8),
                  np.zeros(self.data_size, dtype=np.uint8),
                  np.zeros((self.data_size, 1), dtype=np.int8),
                  np.zeros((self.data_size, 4, 84, 84), dtype=np.uint8),
                  np.zeros((self.data_size, 1), dtype=np.bool)]

    def forward(self, state, action, Reward, state_dash, episode_end):
        num_of_batch = state.shape[0]
        s = Variable(state)
        s_dash = Variable(state_dash)

        Q = self.Q_func(s)  # Get Q-value

        # Generate Target Signals
        max_Q_dash_ = self.Q_func(s_dash)
        tmp = list(map(np.max, max_Q_dash_.data.get()))
        max_Q_dash = np.asanyarray(tmp, dtype=np.float32)
        target = np.asanyarray(Q.data.get(), dtype=np.float32)

        for i in xrange(num_of_batch):
            if not episode_end[i][0]:
                tmp_ = np.sign(Reward[i]) + self.gamma * max_Q_dash[i]
            else:
                tmp_ = np.sign(Reward[i])
            target[i, self.action_to_index(action[i])] = tmp_

        loss = F.mean_squared_error(Variable(cuda.to_gpu(target)), Q)
        return loss, Q

    def stockExperience(self, time,
                        state, action, reward, state_dash,
                        episode_end_flag):
        data_index = time % self.data_size

        if episode_end_flag is True:
            self.D[0][data_index] = state
            self.D[1][data_index] = action
            self.D[2][data_index] = reward
        else:
            self.D[0][data_index] = state
            self.D[1][data_index] = action
            self.D[2][data_index] = reward
            self.D[3][data_index] = state_dash
        self.D[4][data_index] = episode_end_flag

    def experienceReplay(self, time):

        if self.initial_exploration < time:
            # Pick up replay_size number of samples from the Data
            if time < self.data_size:  # during the first sweep of the History Data
                replay_index = np.random.randint(0, time, (self.replay_size, 1))
            else:
                replay_index = np.random.randint(0, self.data_size, (self.replay_size, 1))

            s_replay = np.ndarray(shape=(self.replay_size, 4, 84, 84), dtype=np.float32)
            a_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.uint8)
            r_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.float32)
            s_dash_replay = np.ndarray(shape=(self.replay_size, 4, 84, 84), dtype=np.float32)
            episode_end_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.bool)
            for i in xrange(self.replay_size):
                s_replay[i] = np.asarray(self.D[0][replay_index[i]], dtype=np.float32)
                a_replay[i] = self.D[1][replay_index[i]]
                r_replay[i] = self.D[2][replay_index[i]]
                s_dash_replay[i] = np.array(self.D[3][replay_index[i]], dtype=np.float32)
                episode_end_replay[i] = self.D[4][replay_index[i]]

            s_replay = cuda.to_gpu(s_replay)
            s_dash_replay = cuda.to_gpu(s_dash_replay)
#.........这里部分代码省略.........

# 需要导入模块: from chainer import FunctionSet [as 别名]
# 或者: from chainer.FunctionSet import l1 [as 别名]
class DQN_class:
    # Hyper-Parameters
    gamma = 0.99  # Discount factor
    initial_exploration = 50000  # 10**4  # Initial exploratoin. original: 5x10^4
    replay_size = 32  # Replay (batch) size
    target_model_update_freq = 10 ** 4  # Target update frequancy. original: 10^4
    data_size = 5 * (10 ** 5)  # Data size of history. original: 10^6

    field_num = 7
    field_size = 17

    def __init__(self, control_size=10, field_num=7, field_size=17):
        self.num_of_actions = control_size
        self.field_size = field_size

        # self.enable_controller = enable_controller  # Default setting : "Pong"


        print "Initializing DQN..."
        #	Initialization of Chainer 1.1.0 or older.
        # print "CUDA init"
        # cuda.init()

        self.field_num = field_num

        print "Model Building"
        self.model = FunctionSet(
                l1=F.Convolution2D(self.field_num * 4, 16, ksize=5, stride=1, nobias=False, wscale=np.sqrt(2)),
                l2=F.Convolution2D(16, 24, ksize=4, stride=1, nobias=False, wscale=np.sqrt(2)),
                l3=F.Linear(2400, 512, wscale=np.sqrt(2)),
                q_value=F.Linear(512, self.num_of_actions,
                                 initialW=np.zeros((self.num_of_actions, 512),
                                                   dtype=np.float32))
        ).to_gpu()

        self.model_target = copy.deepcopy(self.model)

        print "Initizlizing Optimizer"
        self.optimizer = optimizers.RMSpropGraves(lr=0.00025, alpha=0.95, momentum=0.95, eps=0.0001)
        # self.optimizer.setup(self.model.collect_parameters())
        self.optimizer.setup(self.model)

        # History Data :  D=[s, a, r, s_dash, end_episode_flag]
        self.D = [np.zeros((self.data_size, self.field_num * 4, self.field_size, self.field_size), dtype=np.uint8),
                  np.zeros(self.data_size, dtype=np.uint8),
                  np.zeros((self.data_size, 1), dtype=np.float32),
                  np.zeros((self.data_size, self.field_num * 4, self.field_size, self.field_size), dtype=np.uint8),
                  np.zeros((self.data_size, 1), dtype=np.bool)]

    def forward(self, state, action, Reward, state_dash, episode_end):
        num_of_batch = state.shape[0]
        s = Variable(state)
        s_dash = Variable(state_dash)

        Q = self.Q_func(s)  # Get Q-value

        # Generate Target Signals
        tmp = self.Q_func_target(s_dash)  # Q(s',*)
        tmp = list(map(np.max, tmp.data.get()))  # max_a Q(s',a)
        max_Q_dash = np.asanyarray(tmp, dtype=np.float32)
        target = np.asanyarray(Q.data.get(), dtype=np.float32)

        for i in xrange(num_of_batch):
            if not episode_end[i][0]:
                tmp_ = np.sign(Reward[i]) + self.gamma * max_Q_dash[i]
            else:
                tmp_ = np.sign(Reward[i])

                # action_index = self.action_to_index(action[i])
            target[i, action[i]] = tmp_

        # TD-error clipping


        td = Variable(cuda.to_gpu(target)) - Q  # TD error
        # td = Variable(target) - Q  # TD error


        td_tmp = td.data + 1000.0 * (abs(td.data) <= 1)  # Avoid zero division
        td_clip = td * (abs(td.data) <= 1) + td / abs(td_tmp) * (abs(td.data) > 1)

        #print "td_data " + str(td_clip.data)

        zero_val = Variable(cuda.to_gpu(np.zeros((self.replay_size, self.num_of_actions), dtype=np.float32)))
        # zero_val = Variable(np.zeros((self.replay_size, self.num_of_actions), dtype=np.float32))



        loss = F.mean_squared_error(td_clip, zero_val)
        return loss, Q

    def stockExperience(self, time,
                        state, action, reward, state_dash,
                        episode_end_flag):
        data_index = time % self.data_size

        if episode_end_flag is True:
            self.D[0][data_index] = state
            self.D[1][data_index] = action
            self.D[2][data_index] = reward
#.........这里部分代码省略.........

# 需要导入模块: from chainer import FunctionSet [as 别名]
# 或者: from chainer.FunctionSet import l1 [as 别名]
class DQN_class:
    # Hyper-Parameters
    gamma = 0.99  # Discount factor
    initial_exploration = 100#10**4  # Initial exploratoin. original: 5x10^4
    replay_size = 32  # Replay (batch) size
    target_model_update_freq = 10**4  # Target update frequancy. original: 10^4
    data_size = 10**5 #10**5  # Data size of history. original: 10^6

    def __init__(self, enable_controller=[0, 3, 4]):
        self.num_of_actions = len(enable_controller)
        self.enable_controller = enable_controller  # Default setting : "Pong"

        print "Initializing DQN..."

        print "Model Building"
        self.CNN_model = FunctionSet(
            l1=F.Convolution2D(4, 32, ksize=8, stride=4, nobias=False, wscale=np.sqrt(2)),
            l2=F.Convolution2D(32, 64, ksize=4, stride=2, nobias=False, wscale=np.sqrt(2)),
            l3=F.Convolution2D(64, 64, ksize=3, stride=1, nobias=False, wscale=np.sqrt(2)),
            )

        self.model = FunctionSet(
            l4=F.Linear(3136, 512, wscale=np.sqrt(2)),
            q_value=F.Linear(512, self.num_of_actions,
                             initialW=np.zeros((self.num_of_actions, 512),
                                               dtype=np.float32))
        ).to_gpu()
        
        d = 'elite/'
        
        self.CNN_model.l1.W.data = np.load(d+'l1_W.npy')#.astype(np.float32)
        self.CNN_model.l1.b.data = np.load(d+'l1_b.npy')#.astype(np.float32)
        self.CNN_model.l2.W.data = np.load(d+'l2_W.npy')#.astype(np.float32)
        self.CNN_model.l2.b.data = np.load(d+'l2_b.npy')#.astype(np.float32)
        self.CNN_model.l3.W.data = np.load(d+'l3_W.npy')#.astype(np.float32)
        self.CNN_model.l3.b.data = np.load(d+'l3_b.npy')#.astype(np.float32)

        self.CNN_model = self.CNN_model.to_gpu()
        self.CNN_model_target = copy.deepcopy(self.CNN_model)
        self.model_target = copy.deepcopy(self.model)


        
        print "Initizlizing Optimizer"
        self.optimizer = optimizers.RMSpropGraves(lr=0.00025, alpha=0.95, momentum=0.95, eps=0.0001)
        self.optimizer.setup(self.model.collect_parameters())

        # History Data :  D=[s, a, r, s_dash, end_episode_flag]
        self.D = [np.zeros((self.data_size, 4, 84, 84), dtype=np.uint8),
                  np.zeros(self.data_size, dtype=np.uint8),
                  np.zeros((self.data_size, 1), dtype=np.int8),
                  np.zeros((self.data_size, 4, 84, 84), dtype=np.uint8),
                  np.zeros((self.data_size, 1), dtype=np.bool),
                  np.zeros((self.data_size, 1), dtype=np.uint8)]
        


    def forward(self, state, action, Reward, state_dash, episode_end):
        num_of_batch = state.shape[0]
        s = Variable(state)
        s_dash = Variable(state_dash)

        Q = self.Q_func(s)  # Get Q-value

        # Generate Target Signals
        tmp = self.Q_func_target(s_dash)  # Q(s',*)
        tmp = list(map(np.max, tmp.data.get()))  # max_a Q(s',a)
        max_Q_dash = np.asanyarray(tmp, dtype=np.float32)
        target = np.asanyarray(Q.data.get(), dtype=np.float32)

        for i in xrange(num_of_batch):
            if not episode_end[i][0]:
                tmp_ = np.sign(Reward[i]) + self.gamma * max_Q_dash[i]
            else:
                tmp_ = np.sign(Reward[i])

            action_index = self.action_to_index(action[i])
            target[i, action_index] = tmp_

        # TD-error clipping
        td = Variable(cuda.to_gpu(target)) - Q  # TD error
        td_tmp = td.data + 1000.0 * (abs(td.data) <= 1)  # Avoid zero division
        td_clip = td * (abs(td.data) <= 1) + td/abs(td_tmp) * (abs(td.data) > 1)

        zero_val = Variable(cuda.to_gpu(np.zeros((self.replay_size, self.num_of_actions), dtype=np.float32)))
        loss = F.mean_squared_error(td_clip, zero_val)
        return loss, Q

    def stockExperience(self, time,
                        state, action, lstm_reward, state_dash,
                        episode_end_flag, ale_reward):
        data_index = time % self.data_size

        if episode_end_flag is True:
            self.D[0][data_index] = state
            self.D[1][data_index] = action
            self.D[2][data_index] = lstm_reward
            self.D[5][data_index] = ale_reward
        else:
            self.D[0][data_index] = state
#.........这里部分代码省略.........

# 需要导入模块: from chainer import FunctionSet [as 别名]
# 或者: from chainer.FunctionSet import l1 [as 别名]
class ChainerAgent(Agent):
	def __init__(self, epsilon=1.0, frames_per_action=4):
		super(ChainerAgent, self).__init__()
		cuda.init()
		self.epsilon = epsilon
		self.gamma = 0.99
		self.iterations = 0
		
		self.model = FunctionSet(
			l1 = F.Linear(9 * frames_per_action, 256),
			l2 = F.Linear(256, 256),
			l3 = F.Linear(256, 256),
			l4 = F.Linear(256, 2),
		).to_gpu()

		self.optimizer = optimizers.RMSprop(lr=1e-5)
		self.optimizer.setup(self.model)
		self.update_target()

		self.num_frames = 0
		self.frames_per_action = frames_per_action
		self.prev_reward = 0.0

		self.history = ChainHistory(state_len=(9 * frames_per_action))

	def forward(self, state, action, reward, new_state, is_terminal):
		q = self.get_q(Variable(state))
		q_target = self.get_target_q(Variable(new_state))

		max_target_q = cp.max(q_target.data, axis=1)

		target = cp.copy(q.data)

		for i in xrange(target.shape[0]):
			curr_action = int(action[i])
			if is_terminal[i]:
				target[i, curr_action] = reward[i]
			else:
				target[i, curr_action] = reward[i] + self.gamma * max_target_q[i]
		
		loss = F.mean_squared_error(Variable(target), q)
		return loss, 0.0 #cp.mean(q.data[:, action[i]])

	def get_q(self, state):
		h1 = F.relu(self.model.l1(state))
		h2 = F.relu(self.model.l2(h1))
		h3 = F.relu(self.model.l3(h2))
		return self.model.l4(h3)

	def get_target_q(self, state):
		h1 = F.relu(self.target_model.l1(state))
		h2 = F.relu(self.target_model.l2(h1))
		h3 = F.relu(self.target_model.l3(h2))
		return self.target_model.l4(h3)

	def accept_reward(self, state, action, reward, new_state, is_terminal):
		self.prev_reward += reward

		if not (is_terminal or self.num_frames % self.frames_per_action == 0):
			return

		if self.num_frames == self.frames_per_action:
			self.prev_reward = 0.0
			self.prev_action = action
			return

		self.history.add((self.prev_state, self.prev_action, self.prev_reward,
			self.curr_state, is_terminal))
		self.prev_reward = 0.0
		self.prev_action = action

		self.iterations += 1
		if self.iterations % 10000 == 0:
			print '*** UPDATING TARGET NETWORK ***'
			self.update_target()
		
		state, action, reward, new_state, is_terminal = self.history.get(num=32)

		state = cuda.to_gpu(state)
		action = cuda.to_gpu(action)
		new_state = cuda.to_gpu(new_state)
		reward = cuda.to_gpu(reward)

		loss, q = self.forward(state, action, reward, new_state, is_terminal)
		self.optimizer.zero_grads()
		loss.backward()
		self.optimizer.update()

	def update_state_vector(self, state):
		if self.num_frames < self.frames_per_action:
			if self.num_frames == 0:
				self.curr_state = state
			else:
				self.curr_state = np.hstack((self.curr_state, state))
		else:
			if self.num_frames < 2 * self.frames_per_action:
				if self.num_frames == self.frames_per_action:
					self.prev_state = np.copy(self.curr_state[:, :9])
				else:
					self.prev_state = np.hstack((self.prev_state, self.curr_state[:, :9]))
#.........这里部分代码省略.........

# 需要导入模块: from chainer import FunctionSet [as 别名]
# 或者: from chainer.FunctionSet import l1 [as 别名]
class SDA(object):
	def __init__(
		self,
		rng,
		data,
		target,
		n_inputs=784,
		n_hidden=[784,784,784],
		n_outputs=10,
		corruption_levels=[0.1,0.2,0.3],
		gpu=-1):

		self.model = FunctionSet(
			l1=F.Linear(n_inputs, n_hidden[0]),
			l2=F.Linear(n_hidden[0], n_hidden[1]),
			l3=F.Linear(n_hidden[1], n_hidden[2]),
			l4=F.Linear(n_hidden[2], n_outputs)
		)

		if gpu >= 0:
			self.model.to_gpu()

		self.rng = rng
		self.gpu = gpu
		self.data = data
		self.target = target

		self.x_train, self.x_test = data
		self.y_train, self.y_test = target

		self.n_train = len(self.y_train)
		self.n_test = len(self.y_test)

		self.corruption_levels = corruption_levels
		self.n_inputs = n_inputs
		self.n_hidden = n_hidden
		self.n_outputs = n_outputs

		self.dae1 = None
		self.dae2 = None
		self.dae3 = None
		self.optimizer = None
		self.setup_optimizer()

		self.train_accuracies = []
		self.train_losses = []

		self.test_accuracies = []
		self.test_losses = []

	def setup_optimizer(self):
		self.optimizer = optimizers.AdaDelta()
		self.optimizer.setup(self.model)

	@property
	def xp(self):
		return cuda.cupy if self.gpu >= 0 else numpy

	def pre_train(self, n_epoch=20, batchsize=100):
		first_inputs = self.data

		# initialize first dAE
		self.dae1 = DA(self.rng,
					   data=first_inputs,
					   n_inputs=self.n_inputs,
					   n_hidden=self.n_hidden[0],
					   corruption_level=self.corruption_levels[0],
					   gpu=self.gpu)
		# train first dAE
		logging.info("--------First DA training has started!--------")
		self.dae1.train_and_test(n_epoch=n_epoch, batchsize=batchsize)
		self.dae1.to_cpu()
		# compute second iputs for second dAE
		tmp1 = self.dae1.compute_hidden(first_inputs[0])
		tmp2 = self.dae1.compute_hidden(first_inputs[1])
		if self.gpu >= 0:
			self.dae1.to_gpu()
		second_inputs = [tmp1, tmp2]

		# initialize second dAE
		self.dae2 = DA(
			self.rng,
			data=second_inputs,
			n_inputs=self.n_hidden[0],
			n_hidden=self.n_hidden[1],
			corruption_level=self.corruption_levels[1],
			gpu=self.gpu
		)
		# train second dAE
		logging.info("--------Second DA training has started!--------")
		self.dae2.train_and_test(n_epoch=n_epoch, batchsize=batchsize)
		self.dae2.to_cpu()
		# compute third inputs for third dAE
		tmp1 = self.dae2.compute_hidden(second_inputs[0])
		tmp2 = self.dae2.compute_hidden(second_inputs[1])
		if self.gpu >= 0:
			self.dae2.to_gpu()
		third_inputs = [tmp1, tmp2]

		# initialize third dAE
#.........这里部分代码省略.........

# 需要导入模块: from chainer import FunctionSet [as 别名]
# 或者: from chainer.FunctionSet import l1 [as 别名]
class DQN_class:
    # Hyper-Parameters
    gamma = 0.99                       # Discount factor
    initial_exploration = 5*10**4      # 10**4  # Initial exploratoin. original: 5x10^4
    replay_size = 32                   # Replay (batch) size
    target_model_update_freq = 10**4   # Target update frequancy. original: 10^4
    data_size = 10**6                  # Data size of history. original: 10^6
    num_of_actions = 2                 # Action dimention
    num_of_states = 12                 # State dimention
    
    def __init__(self):
                  
        print "Initializing DQN..."
#	Initialization of Chainer 1.1.0 or older.
#        print "CUDA init"
#        cuda.init()

        print "Model Building"
#        self.model = FunctionSet(
#            l1=F.Convolution2D(4, 32, ksize=8, stride=4, nobias=False, wscale=np.sqrt(2)),
#            l2=F.Convolution2D(32, 64, ksize=4, stride=2, nobias=False, wscale=np.sqrt(2)),
#            l3=F.Convolution2D(64, 64, ksize=3, stride=1, nobias=False, wscale=np.sqrt(2)),
#            l4=F.Linear(3136, 512, wscale=np.sqrt(2)),
#            q_value=F.Linear(512, self.num_of_actions,
#                             initialW=np.zeros((self.num_of_actions, 512),
#                                               dtype=np.float32))
#        ).to_gpu()
        
#        self.critic = FunctionSet(
#            l1=F.Linear(self.num_of_actions+self.num_of_states,512),
#            l2=F.Linear(512,256),
#            l3=F.Linear(256,128),
#            q_value=F.Linear(128,1,initialW=np.zeros((1,128),dtype=np.float32))
#        ).to_gpu()
#        
#        self.actor = FunctionSet(
#            l1=F.Linear(self.num_of_states,512),
#            l2=F.Linear(512,256),
#            l3=F.Linear(256,128),
#            a_value=F.Linear(128,self.num_of_actions,initialW=np.zeros((1,128),dtype=np.float32))
#        ).to_gpu()
        
        self.critic = FunctionSet(
            l1=F.Linear(self.num_of_actions+self.num_of_states,1024),
            l2=F.Linear(1024,512),
            l3=F.Linear(512,256),
            l4=F.Linear(256,128),
            q_value=F.Linear(128,1,initialW=np.zeros((1,128),dtype=np.float32))
        ).to_gpu()
        
        self.actor = FunctionSet(
            l1=F.Linear(self.num_of_states,1024),
            l2=F.Linear(1024,512),
            l3=F.Linear(512,256),
            l4=F.Linear(256,128),
            a_value=F.Linear(128,self.num_of_actions,initialW=np.zeros((1,128),dtype=np.float32))
        ).to_gpu()
        
#        self.critic = FunctionSet(
#            l1=F.Linear(self.num_of_actions+self.num_of_states,1024,wscale=0.01*math.sqrt(self.num_of_actions+self.num_of_states)),
#            l2=F.Linear(1024,512,wscale=0.01*math.sqrt(1024)),
#            l3=F.Linear(512,256,wscale=0.01*math.sqrt(512)),
#            l4=F.Linear(256,128,wscale=0.01*math.sqrt(256)),
#            q_value=F.Linear(128,1,wscale=0.01*math.sqrt(128))
#        ).to_gpu()
#        
#        self.actor = FunctionSet(
#            l1=F.Linear(self.num_of_states,1024,wscale=0.01*math.sqrt(self.num_of_states)),
#            l2=F.Linear(1024,512,wscale=0.01*math.sqrt(1024)),
#            l3=F.Linear(512,256,wscale=0.01*math.sqrt(512)),
#            l4=F.Linear(256,128,wscale=0.01*math.sqrt(256)),
#            a_value=F.Linear(128,self.num_of_actions,wscale=0.01*math.sqrt(128))
#        ).to_gpu()
        
        self.critic_target = copy.deepcopy(self.critic) 
        self.actor_target = copy.deepcopy(self.actor)
        
        print "Initizlizing Optimizer"
        #self.optim_critic = optimizers.RMSpropGraves(lr=0.0001, alpha=0.95, momentum=0.95, eps=0.0001)
        #self.optim_actor = optimizers.RMSpropGraves(lr=0.0001, alpha=0.95, momentum=0.95, eps=0.0001)
        self.optim_critic = optimizers.Adam(alpha=0.00001)
        self.optim_actor = optimizers.Adam(alpha=0.00001)
        self.optim_critic.setup(self.critic)
        self.optim_actor.setup(self.actor)
        
#        self.optim_critic.add_hook(chainer.optimizer.WeightDecay(0.00001))
#        self.optim_critic.add_hook(chainer.optimizer.GradientClipping(10))
#        self.optim_actor.add_hook(chainer.optimizer.WeightDecay(0.00001))
#        self.optim_actor.add_hook(chainer.optimizer.GradientClipping(10))

        # History Data :  D=[s, a, r, s_dash, end_episode_flag]
        self.D = [np.zeros((self.data_size, self.num_of_states), dtype=np.float32),
                  np.zeros((self.data_size, self.num_of_actions), dtype=np.float32),
                  np.zeros((self.data_size, 1), dtype=np.float32),
                  np.zeros((self.data_size, self.num_of_states), dtype=np.float32),
                  np.zeros((self.data_size, 1), dtype=np.bool)]
                  
#        with open('dqn_dump.json', 'a') as f:
#            json.dump(datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S'), f)
#            f.write('\n')
#.........这里部分代码省略.........

# 需要导入模块: from chainer import FunctionSet [as 别名]
# 或者: from chainer.FunctionSet import l1 [as 别名]
class MLP(Base):
    def __init__(self, data=None, target=None, n_inputs=784, n_hidden=784, n_outputs=10, gpu=-1):
        self.excludes.append('xp')
        self.model = FunctionSet(l1=F.Linear(n_inputs, n_hidden),
                                 l2=F.Linear(n_hidden, n_hidden),
                                 l3=F.Linear(n_hidden, n_outputs))

        if gpu >= 0:
            self.model.to_gpu()
            self.xp = cuda.cupy
        else:
            self.xp = np

        if not data is None:
            self.x_train, self.x_test = data
        else:
            self.x_train, self.y_test = None, None

        if not target is None:
            self.y_train, self.y_test = target
            self.n_train = len(self.y_train)
            self.n_test = len(self.y_test)
        else:
            self.y_train, self.y_test = None, None
            self.n_train = 0
            self.n_test = 0

        self.gpu = gpu
        self.optimizer = optimizers.Adam()
        self.optimizer.setup(self.model)

    def forward(self, x_data, y_data, train=True):
        x, t = Variable(x_data), Variable(y_data)
        h1 = F.dropout(F.relu(self.model.l1(x)), train=train)
        h2 = F.dropout(F.relu(self.model.l2(h1)), train=train)
        y = self.model.l3(h2)
        return F.softmax_cross_entropy(y, t), F.accuracy(y, t)

    def train_and_test(self, n_epoch=20, batchsize=100):
        for epoch in xrange(1, n_epoch+1):
            print 'epoch', epoch

            perm = np.random.permutation(self.n_train)
            sum_accuracy = 0
            sum_loss = 0
            for i in xrange(0, self.n_train, batchsize):
                x_batch = self.xp.asarray(self.x_train[perm[i:i+batchsize]])
                y_batch = self.xp.asarray(self.y_train[perm[i:i+batchsize]])

                real_batchsize = len(x_batch)

                self.optimizer.zero_grads()
                loss, acc = self.forward(x_batch, y_batch)
                loss.backward()
                self.optimizer.update()

                sum_loss += float(cuda.to_cpu(loss.data)) * real_batchsize
                sum_accuracy += float(cuda.to_cpu(acc.data)) * real_batchsize

            print 'train mean loss={}, accuracy={}'.format(sum_loss/self.n_train, sum_accuracy/self.n_train)

            # evalation
            sum_accuracy = 0
            sum_loss = 0
            for i in xrange(0, self.n_test, batchsize):
                x_batch = self.xp.asarray(self.x_test[i:i+batchsize])
                y_batch = self.xp.asarray(self.y_test[i:i+batchsize])

                real_batchsize = len(x_batch)

                loss, acc = self.forward(x_batch, y_batch, train=False)

                sum_loss += float(cuda.to_cpu(loss.data)) * real_batchsize
                sum_accuracy += float(cuda.to_cpu(acc.data)) * real_batchsize

            print 'test mean loss={}, accuracy={}'.format(sum_loss/self.n_test, sum_accuracy/self.n_test)