Python pyarrow.serialize方法代码示例

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import serialize [as 别名]
def dump(obj, file, *, serial_type=None, compress=None, pickle_protocol=None):
    if serial_type is None:
        serial_type = SerialType.ARROW if pyarrow is not None else SerialType.PICKLE
    if compress is None:
        compress = CompressType.NONE
    try:
        if serial_type == SerialType.ARROW:
            serialized = pyarrow.serialize(obj, mars_serialize_context())
            data_size = serialized.total_bytes
            write_file_header(file, file_header(serial_type, SERIAL_VERSION, data_size, compress))
            file = open_compression_file(file, compress)
            serialized.write_to(file)
        else:
            pickle_protocol = pickle_protocol or pickle.HIGHEST_PROTOCOL
            serialized = pickle.dumps(obj, protocol=pickle_protocol)
            data_size = len(serialized)
            write_file_header(file, file_header(serial_type, SERIAL_VERSION, data_size, compress))
            file = open_compression_file(file, compress)
            file.write(serialized)
    finally:
        if compress != CompressType.NONE:
            file.close()
    return 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import serialize [as 别名]
def dumps_pyarrow(obj):
    return pyarrow.serialize(obj).to_buffer() 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import serialize [as 别名]
def dumps(obj):
        """
        Serialize an object.

        Returns:
            Implementation-dependent bytes-like object.
            May not be compatible across different versions of pyarrow.
        """
        import pyarrow as pa
        return pa.serialize(obj).to_buffer() 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import serialize [as 别名]
def serialize(data):
    return pyarrow.serialize(data, mars_serialize_context()) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import serialize [as 别名]
def __init__(self, vineyard_client, session_id, data_key, data_id, mode='w',
                 nbytes=None, packed=False, compress=None, auto_register=True,
                 pin_token=None, handler=None):
        from .objectholder import SharedHolderActor

        logger.debug('create vineyard bytes IO: mode = %s, packed = %s', mode, packed)

        super().__init__(session_id, data_key, mode=mode, handler=handler)
        self._client = vineyard_client
        self._data_id = data_id
        self._components = None
        self._offset = 0
        self._nbytes = nbytes
        self._holder_ref = self._storage_ctx.actor_ctx.actor_ref(SharedHolderActor.default_uid())
        self._compress = compress or dataserializer.CompressType.NONE
        self._packed = packed
        self._auto_register = auto_register
        self._pin_token = pin_token

        block_size = options.worker.copy_block_size

        if self.is_readable:
            logger.debug('bytes io get: session_id = %s, data_key = %s, data_id = %r, type(data_id) = %r',
                         session_id, data_key, data_id, type(data_id))
            data = self._client.get(data_id)

            self._components = pyarrow.serialize(data, dataserializer.mars_serialize_context()).to_components()
            if packed:
                self._buf = ArrowComponentsIO(
                    self._components, 'r', compress_out=compress, block_size=block_size)
            else:
                raise NotImplementedError('Unknown how to read vineyard values in a unpacked way')
        else:
            raise NotImplementedError 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import serialize [as 别名]
def get_objects(self, session_id, data_keys, serialize=False, _promise=False):
        data_ids = [self._get_object_id(session_id, data_key) for data_key in data_keys]
        return self._client.get_object(data_ids) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import serialize [as 别名]
def load_from_bytes_io(self, session_id, data_keys, src_handler, pin_token=None):
        def _read_serialized(reader):
            with reader:
                return reader.get_io_pool().submit(reader.read).result()

        def _fallback(*_):
            return self._batch_load_objects(
                session_id, data_keys,
                lambda k: src_handler.create_bytes_reader(session_id, k, _promise=True).then(_read_serialized),
                serialize=True
            )

        return self.transfer_in_runner(session_id, data_keys, src_handler, _fallback) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import serialize [as 别名]
def __init__(self, plasma_client, mapper_ref):
        from ...serialize.dataserializer import mars_serialize_context

        self._plasma_client = plasma_client
        self._actual_size = None
        self._serialize_context = mars_serialize_context()

        self._mapper_ref = mapper_ref
        self._pool = mapper_ref.ctx.threadpool(1) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import serialize [as 别名]
def ray_compress(data):
    data = pyarrow.serialize(data).to_buffer().to_pybytes()
    data = lz4.frame.compress(data)
    # Unclear why ascii decoding.
    data = base64.b64encode(data).decode("ascii")
    # data = base64.b64encode(data)
    return data 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import serialize [as 别名]
def __init__(self, *args, **kwargs):
        super(LocalDiskArrowTableCache, self).__init__(*args, **kwargs)
        # Workaround for https://issues.apache.org/jira/browse/ARROW-5260
        # unless we try to serialize something before deserialize_components is called, we would crash with a sigsegv
        pa.serialize(0) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import serialize [as 别名]
def get(self, key, fill_cache_func):
        value = self._cache.get(key, default=None)
        if value is None:
            value = fill_cache_func()
            table_pandas = value.to_pandas()
            serialized_df = pa.serialize(table_pandas)
            components = serialized_df.to_components()
            self._cache.set(key, components)
        else:
            original_df = pa.deserialize_components(value)
            value = pa.Table.from_pandas(original_df, preserve_index=False)

        return value 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import serialize [as 别名]
def serialize(self, rows):
        return pyarrow.serialize(rows, self._get_serialization_context()).to_buffer() 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import serialize [as 别名]
def send_data_to_buffer(self, replay_data):
        """Send replay data to global buffer."""
        replay_data_id = pa.serialize(replay_data).to_buffer()
        self.push_socket.send(replay_data_id) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import serialize [as 别名]
def send_new_priorities(self, indices: np.ndarray, priorities: np.ndarray):
        """Send new priority values and corresponding indices to buffer."""
        new_priors = [indices, priorities]
        new_priors_id = pa.serialize(new_priors).to_buffer()
        self.rep_socket.send(new_priors_id) 

# 需要导入模块: import pyarrow [as 别名]
# 或者: from pyarrow import serialize [as 别名]
def publish_params(self, update_step: int, np_state_dict: List[np.ndarray]):
        """Broadcast updated params to all workers."""
        param_info = [update_step, np_state_dict]
        new_params_id = pa.serialize(param_info).to_buffer()
        self.pub_socket.send(new_params_id)