Python Connector.assign_npa方法代码示例

# 需要导入模块: from quagga.connector import Connector [as 别名]
# 或者: from quagga.connector.Connector import assign_npa [as 别名]
class DataBlock(object):
    def __init__(self, char_to_idx, device_id):
        self.context = Context(device_id)
        device_id = self.context.device_id
        self.char_idx = Connector(Matrix.empty(1, 1, 'int', device_id))
        self.char_to_idx = char_to_idx
        self.char = None

    def fprop(self):
        char_npa = np.zeros((1, 1), np.int32, 'F')
        char_npa[0][0] = self.char_to_idx[self.char] if self.char in self.char_to_idx else self.char_to_idx['<unk>']
        self.char_idx.assign_npa(self.context, char_npa)
        self.char_idx.fprop()

# 需要导入模块: from quagga.connector import Connector [as 别名]
# 或者: from quagga.connector.Connector import assign_npa [as 别名]
class DataBlock(object):
    def __init__(self, word_to_idx, device_id):
        self.context = Context(device_id)
        device_id = self.context.device_id
        self.word_idx = Connector(Matrix.empty(1, 1, 'int', device_id))
        self.word_to_idx = word_to_idx
        self.word = None

    def fprop(self):
        word_npa = np.zeros((1, 1), np.int32, 'F')
        word_npa[0][0] = self.word_to_idx[self.word] if self.word in self.word_to_idx else self.word_to_idx['<UNK>']
        self.word_idx.assign_npa(self.context, word_npa)
        self.word_idx.fprop()

# 需要导入模块: from quagga.connector import Connector [as 别名]
# 或者: from quagga.connector.Connector import assign_npa [as 别名]
class DataBlock(object):
    def __init__(self, data, char_to_idx, batch_size, x_device_id, y_device_id):
        self.data = HomogeneousDataIterator(data, char_to_idx, batch_size, True, True)
        self.data_iterator = iter(self.data)
        self.x_context = Context(x_device_id)
        self.y_context = Context(y_device_id)
        max_len = 0
        for sub_line in data:
            cur_len = len(sub_line)
            if cur_len > max_len:
                max_len = cur_len
        print max_len
        self.x = Connector(Matrix.empty(batch_size, max_len - 1, 'int', x_device_id))
        self._y = Matrix.empty(batch_size, max_len - 1, 'int', y_device_id)
        self.y = List([Connector(self._y[:, i]) for i in xrange(max_len - 1)], self.x.ncols)
        self.lengths = Matrix.empty(self.x.nrows, 1, 'int', x_device_id)
        self._mask = Matrix.empty(self.x.nrows, self.x.ncols, 'float', x_device_id)
        self.mask = List([Connector(self._mask[:, i]) for i in xrange(max_len)], self.x.ncols)
        self.blocking_contexts = None

    def fprop(self):
        self.x_context.wait(*self.blocking_contexts)
        self.y_context.wait(*self.blocking_contexts)
        data = next(self.data_iterator)
        lengths_npa = np.array([[len(e) - 1] for e in data], np.int32, order='F')
        x_npa = np.zeros((len(data), int(np.max(lengths_npa))), np.int32, 'F')
        for k, e in enumerate(data):
            x_npa[k, :len(e) - 1] = e[:-1]
        self.x.assign_npa(self.x_context, x_npa)
        y_npa = np.zeros((len(data), int(np.max(lengths_npa))), np.int32, 'F')
        for k, e in enumerate(data):
            y_npa[k, :len(e) - 1] = e[1:]
        self._y.assign_npa(self.y_context, y_npa)
        for e in self.y:
            e.last_modification_context = self.y_context
        self.lengths.assign_npa(self.x_context, lengths_npa)
        self._mask.mask_column_numbers_row_wise(self.x_context, self.lengths)
        for e in self.mask:
            e.last_modification_context = self.x_context
        self.x.fprop()
        self.y.fprop()
        self.mask.fprop()

# 需要导入模块: from quagga.connector import Connector [as 别名]
# 或者: from quagga.connector.Connector import assign_npa [as 别名]
class DataBlock(object):
    def __init__(self, train_data, valid_data, batch_size, word_dropout_prob, device_id):
        self.train_data = HomogeneousDataIterator(train_data, batch_size, randomize=True, infinite=True)
        self.valid_data = HomogeneousDataIterator(valid_data, batch_size)
        self.train_data_iterator = iter(self.train_data)
        self.valid_data_iterator = iter(self.valid_data)
        self.word_keep_prob = 1.0 - word_dropout_prob
        self.rnd = RandomState(47571)
        self.unk_idx = word_to_idx['<UNK>']

        self.context = Context(device_id)
        c = Counter([len(line) for line in chain(train_data, valid_data)])
        print c.most_common()
        max_len = max([len(line) for line in chain(train_data, valid_data)])

        self.enc_x = Connector(Matrix.empty(batch_size, max_len, 'int', device_id))
        self.enc_lengths = Matrix.empty(self.enc_x.nrows, 1, 'int', device_id)
        self._enc_mask = Matrix.empty(self.enc_x.nrows, self.enc_x.ncols, 'float', device_id)
        self.enc_mask = List([Connector(self._enc_mask[:, i]) for i in xrange(max_len)], self.enc_x.ncols)

        self.dec_x = Connector(Matrix.empty(batch_size, max_len + 1, 'int', device_id))
        self._dec_y = Matrix.empty(batch_size, max_len + 1, 'int', device_id)
        self.dec_y = List([Connector(self._dec_y[:, i]) for i in xrange(max_len + 1)], self._dec_y.ncols)
        self.dec_lengths = Matrix.empty(self.dec_x.nrows, 1, 'int', device_id)
        self._dec_mask = Matrix.empty(self.dec_x.nrows, self.dec_x.ncols, 'float', device_id)
        self.dec_mask = List([Connector(self._dec_mask[:, i]) for i in xrange(max_len + 1)], self.dec_x.ncols)

        self.blocking_contexts = None
        self.training_mode = True

    def set_training_mode(self):
        self.training_mode = True

    def set_testing_mode(self):
        self.training_mode = False

    def fprop(self):
        if self.training_mode:
            data = next(self.train_data_iterator)
        else:
            try:
                data = next(self.valid_data_iterator)
            except StopIteration as e:
                self.valid_data_iterator = iter(self.valid_data)
                raise e
        lengths_npa = np.array([[len(e)] for e in data], np.int32, order='F')
        max_len = int(np.max(lengths_npa))

        self.enc_lengths.assign_npa(self.context, lengths_npa)
        self._enc_mask.mask_column_numbers_row_wise(self.context, self.enc_lengths)
        for e in self.enc_mask:
            e.last_modification_context = self.context

        lengths_npa += 1
        self.dec_lengths.assign_npa(self.context, lengths_npa)
        self._dec_mask.mask_column_numbers_row_wise(self.context, self.dec_lengths)
        for e in self.dec_mask:
            e.last_modification_context = self.context

        enc_x_npa = np.zeros((len(data), max_len), np.int32, 'F')
        dec_x_npa = np.zeros((len(data), max_len + 1), np.int32, 'F')
        dec_y_npa = np.zeros((len(data), max_len + 1), np.int32, 'F')
        for k, e in enumerate(data):
            enc_x_npa[k, :len(e)] = e
            if self.training_mode:
                new_e = [_ if self.rnd.rand() < self.word_keep_prob else self.unk_idx for _ in e]
            else:
                new_e = e
            dec_x_npa[k, :len(e) + 1] = [word_to_idx['<<S>>']] + new_e
            dec_y_npa[k, :len(e) + 1] = e + [word_to_idx['<<S>>']]
        self.enc_x.assign_npa(self.context, enc_x_npa)
        self.dec_x.assign_npa(self.context, dec_x_npa)
        self._dec_y.assign_npa(self.context, dec_y_npa)
        for e in self.dec_y:
            e.last_modification_context = self.context

        self.enc_mask.fprop()
        self.dec_mask.fprop()
        self.enc_x.fprop()
        self.dec_x.fprop()
        self.dec_y.fprop()