Python pyarrow.deserialize方法代码示例

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import deserialize [as 别名]
def loads(buf):
    mv = memoryview(buf)
    header = read_file_header(mv)
    compress = header.compress

    if compress == CompressType.NONE:
        data = buf[HEADER_LENGTH:]
    else:
        data = decompressors[compress](mv[HEADER_LENGTH:])

    if header.type == SerialType.ARROW:
        try:
            return pyarrow.deserialize(memoryview(data), mars_serialize_context())
        except pyarrow.lib.ArrowInvalid:  # pragma: no cover
            # reconstruct value from buffers of arrow components
            data_view = memoryview(data)
            meta_block_size = np.frombuffer(data_view[0:4], dtype='int32').item()
            meta = pickle.loads(data_view[4:4 + meta_block_size])  # nosec
            buffer_sizes = meta.pop('buffer_sizes')
            bounds = np.cumsum([4 + meta_block_size] + buffer_sizes)
            meta['data'] = [pyarrow.py_buffer(data_view[bounds[idx]:bounds[idx + 1]])
                            for idx in range(len(buffer_sizes))]
            return pyarrow.deserialize_components(meta, mars_serialize_context())
    else:
        return pickle.loads(data) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import deserialize [as 别名]
def recv_params_from_learner(self):
        """Get new params and sync. return True if success, False otherwise."""
        received = False
        try:
            new_params_id = self.sub_socket.recv(zmq.DONTWAIT)
            received = True
        except zmq.Again:
            # Although learner doesn't send params, don't wait
            pass

        if received:
            new_param_info = pa.deserialize(new_params_id)
            update_step, new_params = new_param_info
            self.update_step = update_step
            self.worker.synchronize(new_params)

            # Add new entry for scores dict
            self.scores[self.update_step] = [] 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import deserialize [as 别名]
def run(self):
        """Run main logging loop; continuously receive data and log."""
        if self.args.log:
            self.set_wandb()

        while self.update_step < self.args.max_update_step:
            self.recv_log_info()
            if self.log_info_queue:  # if non-empty
                log_info_id = self.log_info_queue.pop()
                log_info = pa.deserialize(log_info_id)
                state_dict = log_info["state_dict"]
                log_value = log_info["log_value"]
                self.update_step = log_value["update_step"]

                self.synchronize(state_dict)
                avg_score = self.test(self.update_step)
                log_value["avg_score"] = avg_score
                self.write_log(log_value) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import deserialize [as 别名]
def recv_worker_data(self):
        """Receive replay data from worker and incorporate to buffer."""
        received = False
        try:
            new_replay_data_id = self.pull_socket.recv(zmq.DONTWAIT)
            received = True
        except zmq.Again:
            pass

        if received:
            new_replay_data = pa.deserialize(new_replay_data_id)
            experience, priorities = new_replay_data
            for idx in range(len(experience["states"])):
                transition = (
                    experience["states"][idx],
                    experience["actions"][idx],
                    experience["rewards"][idx],
                    experience["next_states"][idx],
                    experience["dones"][idx],
                )
                self.buffer.add(transition)
                self.buffer.update_priorities([len(self.buffer) - 1], priorities[idx]) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import deserialize [as 别名]
def load(self, data):
        # If the type of the input is bytes, then we need to deserialize the
        # input first.
        if type(data) == bytes:
            val = Value()
            val.ParseFromString(data)
        elif type(data).__name__ == Value.__name__:
            # If it's already deserialized, we can just proceed.
            val = data
        else:
            raise ValueError(f'''Input to load was of unsupported type
                             {str(type(data))}.''')

        if val.type == DEFAULT:
            try:
                return self._load_default(val.body)
            except: # Unpickling error.
                return val.body
        elif val.type == STRING:
            return self._load_string(val.body)
        elif val.type == NUMPY:
            return self._load_numpy(val.body) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import deserialize [as 别名]
def load_pyarrow(buf):
    assert buf is not None, 'buf should not be None.'
    return pyarrow.deserialize(buf) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import deserialize [as 别名]
def loads(buf):
        """
        Args:
            buf: the output of `dumps` or `dumps_bytes`.
        """
        import pyarrow as pa
        return pa.deserialize(buf) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import deserialize [as 别名]
def load(file):
    header = read_file_header(file)
    file = open_decompression_file(file, header.compress)

    try:
        buf = file.read()
    finally:
        if header.compress != CompressType.NONE:
            file.close()

    if header.type == SerialType.ARROW:
        return pyarrow.deserialize(memoryview(buf), mars_serialize_context())
    else:
        return pickle.loads(buf) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import deserialize [as 别名]
def deserialize(data):
    return pyarrow.deserialize(data, mars_serialize_context()) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import deserialize [as 别名]
def ray_decompress(data):
    if isinstance(data, bytes) or isinstance(data, string_types):
        data = base64.b64decode(data)
        data = lz4.frame.decompress(data)
        data = pyarrow.deserialize(data)
    return data


# Ray's magic constant worker explorations.. 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import deserialize [as 别名]
def deserialize(self, serialized_rows):
        return pyarrow.deserialize(serialized_rows, self._get_serialization_context()) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import deserialize [as 别名]
def recv_replay_data(self):
        """Receive replay data from gloal buffer."""
        replay_data_id = self.rep_socket.recv()
        replay_data = pa.deserialize(replay_data_id)
        return replay_data 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import deserialize [as 别名]
def send_batch_to_learner(self):
        """Send batch to learner and receive priorities."""
        # Send batch and request priorities (blocking recv)
        batch = self.buffer.sample(self.per_beta)
        batch_id = pa.serialize(batch).to_buffer()
        self.req_socket.send(batch_id)
        self.num_sent = self.num_sent + 1

        # Receive priorities
        new_priors_id = self.req_socket.recv()
        idxes, new_priorities = pa.deserialize(new_priors_id)
        self.buffer.update_priorities(idxes, new_priorities) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import deserialize [as 别名]
def loads_pyarrow(buf):
    """
    Args:
        buf: the output of `dumps`.
    """
    return pa.deserialize(buf) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import deserialize [as 别名]
def loads_pyarrow(buf):
    """
    Args:
        buf: the output of `dumps`.
    """
    return pa.deserialize(buf)


# import pyarrow has a lot of side effect:
# https://github.com/apache/arrow/pull/2329
# https://groups.google.com/a/tensorflow.org/forum/#!topic/developers/TMqRaT-H2bI
# So we use msgpack as default.