Python multiprocessing.RawArray方法代码示例

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import RawArray [as 别名]
def extract_params_as_shared_arrays(link):
    assert isinstance(link, chainer.Link)
    shared_arrays = {}
    for param_name, param in link.namedparams():
        typecode = param.array.dtype.char
        shared_arrays[param_name] = mp.RawArray(typecode, param.array.ravel())

    for persistent_name, persistent in chainerrl.misc.namedpersistent(link):
        if isinstance(persistent, np.ndarray):
            typecode = persistent.dtype.char
            shared_arrays[persistent_name] = mp.RawArray(
                typecode, persistent.ravel())
        else:
            assert np.isscalar(persistent)
            # Wrap by a 1-dim array because multiprocessing.RawArray does not
            # accept a 0-dim array.
            persistent_as_array = np.asarray([persistent])
            typecode = persistent_as_array.dtype.char
            shared_arrays[persistent_name] = mp.RawArray(
                typecode, persistent_as_array)
    return shared_arrays 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import RawArray [as 别名]
def borrow_all_memories(shared_params, memshared_instances):
    """
    Run theano_borrow_memory on a list of params and shared memory
    sharedctypes.

    Parameters
    ----------
    shared_params : list
        of :class:`theano.tensor.sharedvar.TensorSharedVariable`
        the Theano shared variable where
        shared memory should be used instead.
    memshared_instances : dict of tuples
        of :class:`multiprocessing.RawArray` and their shapes
        the memory shared across processes (e.g.from `memshare_sparams`)

    Notes
    -----
    Same as `borrow_memory` but for lists of shared memories and
    theano variables. See `borrow_memory`
    """
    for sparam in shared_params:
        borrow_memory(sparam, *memshared_instances[sparam.name]) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import RawArray [as 别名]
def test_array(self, raw=False):
        seq = [680, 626, 934, 821, 150, 233, 548, 982, 714, 831]
        if raw:
            arr = self.RawArray('i', seq)
        else:
            arr = self.Array('i', seq)

        self.assertEqual(len(arr), len(seq))
        self.assertEqual(arr[3], seq[3])
        self.assertEqual(list(arr[2:7]), list(seq[2:7]))

        arr[4:8] = seq[4:8] = array.array('i', [1, 2, 3, 4])

        self.assertEqual(list(arr[:]), seq)

        self.f(seq)

        p = self.Process(target=self.f, args=(arr,))
        p.daemon = True
        p.start()
        p.join()

        self.assertEqual(list(arr[:]), seq) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import RawArray [as 别名]
def __new__(cls, shape, dtype=None, buffer=None, offset=None, strides=None,
                order=None):
        # init buffer
        if buffer is None:
            assert offset is None
            assert strides is None
            size = int(np.prod(shape))
            nbytes = size * np.dtype(dtype).itemsize
            # this is the part that can be passed between processes
            shmem = mp.RawArray(ctypes.c_char, nbytes)
            offset = 0
        elif isinstance(buffer, ctypes.Array):
            # restoring from a pickle
            shmem = buffer
        else:
            raise ValueError(
                f"{cls.__name__} does not support specifying custom "
                f" buffers, but was given {buffer!r}")

        # init array
        obj = np.ndarray.__new__(cls, shape, dtype=dtype, buffer=shmem,
                                 offset=offset, strides=strides, order=order)
        obj._shmem = shmem

        return obj 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import RawArray [as 别名]
def mpraw_as_np(shape, dtype):
    """Construct a numpy array of the specified shape and dtype for which the
    underlying storage is a multiprocessing RawArray in shared memory.

    Parameters
    ----------
    shape : tuple
      Shape of numpy array
    dtype : data-type
      Data type of array

    Returns
    -------
    arr : ndarray
      Numpy array
    """

    sz = int(np.product(shape))
    csz = sz * np.dtype(dtype).itemsize
    raw = mp.RawArray('c', csz)
    return np.frombuffer(raw, dtype=dtype, count=sz).reshape(shape) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import RawArray [as 别名]
def mpraw_as_np(shape, dtype):
    """Construct a numpy array of the specified shape and dtype for
    which the underlying storage is a multiprocessing RawArray in shared
    memory.

    Parameters
    ----------
    shape : tuple
      Shape of numpy array
    dtype : data-type
      Data type of array

    Returns
    -------
    arr : ndarray
      Numpy array
    """

    sz = int(np.product(shape))
    csz = sz * np.dtype(dtype).itemsize
    raw = mp.RawArray('c', csz)
    return np.frombuffer(raw, dtype=dtype, count=sz).reshape(shape) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import RawArray [as 别名]
def share_array(arr_np):
    """
    Return a ctypes array allocated from shared memory with data from a
    NumPy array of type `float`.

    Parameters
    ----------
            arr_np : :obj:`numpy.ndarray`
                    NumPy array.

    Returns
    -------
            array of :obj:`ctype`
    """

    arr = mp.RawArray('d', arr_np.ravel())
    return arr, arr_np.shape 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import RawArray [as 别名]
def _share_array(arr_np, typecode_or_type):
    """
    Return a ctypes array allocated from shared memory with data from a
    NumPy array.

    Parameters
    ----------
        arr_np : :obj:`numpy.ndarray`
            NumPy array.
        typecode_or_type : char or :obj:`ctype`
            Either a ctypes type or a one character typecode of the
            kind used by the Python array module.

    Returns
    -------
        array of :obj:`ctype`
    """

    arr = mp.RawArray(typecode_or_type, arr_np.ravel())
    return arr, arr_np.shape 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import RawArray [as 别名]
def _create_shared_arr(self):
        TYPE = {
            np.float32: ctypes.c_float,
            np.float64: ctypes.c_double,
            np.uint8: ctypes.c_uint8,
            np.int8: ctypes.c_int8,
            np.int32: ctypes.c_int32,
        }
        ctype = TYPE[self.output_dtype]
        arr = mp.RawArray(ctype, int(np.prod(self.output_shape)))
        return arr 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import RawArray [as 别名]
def extract_states_as_shared_arrays(optimizer):
    assert isinstance(optimizer, chainer.Optimizer)
    assert hasattr(optimizer, 'target'), 'Optimizer.setup must be called first'
    shared_arrays = {}
    for param_name, param in optimizer.target.namedparams():
        shared_arrays[param_name] = {}
        ensure_initialized_update_rule(param)
        state = param.update_rule.state
        for state_name, state_val in state.items():
            shared_arrays[param_name][
                state_name] = mp.RawArray('f', state_val.ravel())
    return shared_arrays 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import RawArray [as 别名]
def extract_params_as_shared_arrays(link):
    assert isinstance(link, chainer.Link)
    shared_arrays = {}
    for param_name, param in link.namedparams():
        shared_arrays[param_name] = mp.RawArray('f', param.data.ravel())
    return shared_arrays 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import RawArray [as 别名]
def extract_states_as_shared_arrays(optimizer):
    assert isinstance(optimizer, chainer.Optimizer)
    assert hasattr(optimizer, 'target'), 'Optimizer.setup must be called first'
    shared_arrays = {}
    for state_name, state in optimizer._states.items():
        shared_arrays[state_name] = {}
        for param_name, param in state.items():
            shared_arrays[state_name][
                param_name] = mp.RawArray('f', param.ravel())
    return shared_arrays 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import RawArray [as 别名]
def __init__(self, *, slot_bytes, slot_count):
        """Initializer.

        Args:
            slot_bytes: How big each buffer in the array should be.
            slot_count: How many buffers should be in the array.
        """
        self.slot_bytes = slot_bytes
        self.slot_count = slot_count
        self.length_bytes = 4
        slot_type = ctypes.c_byte * (slot_bytes + self.length_bytes)
        self.array = multiprocessing.RawArray(slot_type, slot_count) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import RawArray [as 别名]
def malloc_contiguous(self, size, initial_val=None):
        if initial_val is None:
            return RawArray(ctypes.c_float, size)
        else:
            return RawArray(ctypes.c_float, initial_val) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import RawArray [as 别名]
def __init__(self, num_actors):
        self.updated = RawArray(ctypes.c_int, num_actors)