Python MemmapingPool.terminate方法代码示例

# 需要导入模块: from joblib.pool import MemmapingPool [as 别名]
# 或者: from joblib.pool.MemmapingPool import terminate [as 别名]
def test_memmaping_on_dev_shm():
    """Check that MemmapingPool uses /dev/shm when possible"""
    p = MemmapingPool(3, max_nbytes=10)
    try:
        # Check that the pool has correctly detected the presence of the
        # shared memory filesystem.
        pool_temp_folder = p._temp_folder
        folder_prefix = '/dev/shm/joblib_memmaping_pool_'
        assert pool_temp_folder.startswith(folder_prefix)
        assert os.path.exists(pool_temp_folder)

        # Try with a file larger than the memmap threshold of 10 bytes
        a = np.ones(100, dtype=np.float64)
        assert a.nbytes == 800
        p.map(id, [a] * 10)
        # a should have been memmaped to the pool temp folder: the joblib
        # pickling procedure generate one .pkl file:
        assert len(os.listdir(pool_temp_folder)) == 1

        # create a new array with content that is different from 'a' so that
        # it is mapped to a different file in the temporary folder of the
        # pool.
        b = np.ones(100, dtype=np.float64) * 2
        assert b.nbytes == 800
        p.map(id, [b] * 10)
        # A copy of both a and b are now stored in the shared memory folder
        assert len(os.listdir(pool_temp_folder)) == 2

    finally:
        # Cleanup open file descriptors
        p.terminate()
        del p

    # The temp folder is cleaned up upon pool termination
    assert not os.path.exists(pool_temp_folder)

# 需要导入模块: from joblib.pool import MemmapingPool [as 别名]
# 或者: from joblib.pool.MemmapingPool import terminate [as 别名]
def test_pool_with_memmap_array_view(tmpdir):
    """Check that subprocess can access and update shared memory array"""
    assert_array_equal = np.testing.assert_array_equal

    # Fork the subprocess before allocating the objects to be passed
    pool_temp_folder = tmpdir.mkdir('pool').strpath
    p = MemmapingPool(10, max_nbytes=2, temp_folder=pool_temp_folder)
    try:

        filename = tmpdir.join('test.mmap').strpath
        a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+')
        a.fill(1.0)

        # Create an ndarray view on the memmap instance
        a_view = np.asarray(a)
        assert not isinstance(a_view, np.memmap)
        assert has_shareable_memory(a_view)

        p.map(inplace_double, [(a_view, (i, j), 1.0)
                               for i in range(a.shape[0])
                               for j in range(a.shape[1])])

        # Both a and the a_view have been updated
        assert_array_equal(a, 2 * np.ones(a.shape))
        assert_array_equal(a_view, 2 * np.ones(a.shape))

        # Passing memmap array view to the pool should not trigger the
        # creation of new files on the FS
        assert os.listdir(pool_temp_folder) == []

    finally:
        p.terminate()
        del p

# 需要导入模块: from joblib.pool import MemmapingPool [as 别名]
# 或者: from joblib.pool.MemmapingPool import terminate [as 别名]
def test_memmaping_on_dev_shm():
    """Check that MemmapingPool uses /dev/shm when possible"""
    p = MemmapingPool(3, max_nbytes=10)
    try:
        # Check that the pool has correctly detected the presence of the
        # shared memory filesystem.
        pool_temp_folder = p._temp_folder
        folder_prefix = '/dev/shm/joblib_memmaping_pool_'
        assert_true(pool_temp_folder.startswith(folder_prefix))
        assert_true(os.path.exists(pool_temp_folder))

        # Try with a file larger than the memmap threshold of 10 bytes
        a = np.ones(100, dtype=np.float64)
        assert_equal(a.nbytes, 800)
        p.map(id, [a] * 10)
        # a should have been memmaped to the pool temp folder: the joblib
        # pickling procedure generate a .pkl and a .npy file:
        assert_equal(len(os.listdir(pool_temp_folder)), 2)

        b = np.ones(100, dtype=np.float64)
        assert_equal(b.nbytes, 800)
        p.map(id, [b] * 10)
        # A copy of both a and b are not stored in the shared memory folder
        assert_equal(len(os.listdir(pool_temp_folder)), 4)

    finally:
        # Cleanup open file descriptors
        p.terminate()
        del p

    # The temp folder is cleaned up upon pool termination
    assert_false(os.path.exists(pool_temp_folder))

# 需要导入模块: from joblib.pool import MemmapingPool [as 别名]
# 或者: from joblib.pool.MemmapingPool import terminate [as 别名]
def test_pool_with_memmap(tmpdir_path):
    """Check that subprocess can access and update shared memory memmap"""
    assert_array_equal = np.testing.assert_array_equal

    # Fork the subprocess before allocating the objects to be passed
    pool_temp_folder = os.path.join(tmpdir_path, 'pool')
    os.makedirs(pool_temp_folder)
    p = MemmapingPool(10, max_nbytes=2, temp_folder=pool_temp_folder)
    try:
        filename = os.path.join(tmpdir_path, 'test.mmap')
        a = np.memmap(filename, dtype=np.float32, shape=(3, 5), mode='w+')
        a.fill(1.0)

        p.map(inplace_double, [(a, (i, j), 1.0)
                               for i in range(a.shape[0])
                               for j in range(a.shape[1])])

        assert_array_equal(a, 2 * np.ones(a.shape))

        # Open a copy-on-write view on the previous data
        b = np.memmap(filename, dtype=np.float32, shape=(5, 3), mode='c')

        p.map(inplace_double, [(b, (i, j), 2.0)
                               for i in range(b.shape[0])
                               for j in range(b.shape[1])])

        # Passing memmap instances to the pool should not trigger the creation
        # of new files on the FS
        assert os.listdir(pool_temp_folder) == []

        # the original data is untouched
        assert_array_equal(a, 2 * np.ones(a.shape))
        assert_array_equal(b, 2 * np.ones(b.shape))

        # readonly maps can be read but not updated
        c = np.memmap(filename, dtype=np.float32, shape=(10,), mode='r',
                      offset=5 * 4)

        assert_raises(AssertionError, p.map, check_array,
                      [(c, i, 3.0) for i in range(c.shape[0])])

        # depending on the version of numpy one can either get a RuntimeError
        # or a ValueError
        assert_raises((RuntimeError, ValueError), p.map, inplace_double,
                      [(c, i, 2.0) for i in range(c.shape[0])])
    finally:
        # Clean all filehandlers held by the pool
        p.terminate()
        del p

# 需要导入模块: from joblib.pool import MemmapingPool [as 别名]
# 或者: from joblib.pool.MemmapingPool import terminate [as 别名]
def test_memmaping_pool_for_large_arrays():
    """Check that large arrays are not copied in memory"""
    assert_array_equal = np.testing.assert_array_equal

    # Check that the tempfolder is empty
    assert_equal(os.listdir(TEMP_FOLDER), [])

    # Build an array reducers that automaticaly dump large array content
    # to filesystem backed memmap instances to avoid memory explosion
    p = MemmapingPool(3, max_nbytes=40, temp_folder=TEMP_FOLDER)
    try:
        # The tempory folder for the pool is not provisioned in advance
        assert_equal(os.listdir(TEMP_FOLDER), [])
        assert_false(os.path.exists(p._temp_folder))

        small = np.ones(5, dtype=np.float32)
        assert_equal(small.nbytes, 20)
        p.map(check_array, [(small, i, 1.0) for i in range(small.shape[0])])

        # Memory has been copied, the pool filesystem folder is unused
        assert_equal(os.listdir(TEMP_FOLDER), [])

        # Try with a file larger than the memmap threshold of 40 bytes
        large = np.ones(100, dtype=np.float64)
        assert_equal(large.nbytes, 800)
        p.map(check_array, [(large, i, 1.0) for i in range(large.shape[0])])

        # The data has been dumped in a temp folder for subprocess to share it
        # without per-child memory copies
        assert_true(os.path.isdir(p._temp_folder))
        dumped_filenames = os.listdir(p._temp_folder)
        assert_equal(len(dumped_filenames), 2)

        # Check that memmory mapping is not triggered for arrays with
        # dtype='object'
        objects = np.array(['abc'] * 100, dtype='object')
        results = p.map(has_shareable_memory, [objects])
        assert_false(results[0])

    finally:
        # check FS garbage upon pool termination
        p.terminate()
        assert_false(os.path.exists(p._temp_folder))
        del p

# 需要导入模块: from joblib.pool import MemmapingPool [as 别名]
# 或者: from joblib.pool.MemmapingPool import terminate [as 别名]
def test_memmaping_pool_for_large_arrays_in_return(tmpdir):
    """Check that large arrays are not copied in memory in return"""
    assert_array_equal = np.testing.assert_array_equal

    # Build an array reducers that automaticaly dump large array content
    # but check that the returned datastructure are regular arrays to avoid
    # passing a memmap array pointing to a pool controlled temp folder that
    # might be confusing to the user

    # The MemmapingPool user can always return numpy.memmap object explicitly
    # to avoid memory copy
    p = MemmapingPool(3, max_nbytes=10, temp_folder=tmpdir.strpath)
    try:
        res = p.apply_async(np.ones, args=(1000,))
        large = res.get()
        assert not has_shareable_memory(large)
        assert_array_equal(large, np.ones(1000))
    finally:
        p.terminate()
        del p

# 需要导入模块: from joblib.pool import MemmapingPool [as 别名]
# 或者: from joblib.pool.MemmapingPool import terminate [as 别名]
def test_memmaping_pool_for_large_arrays_disabled(tmpdir):
    """Check that large arrays memmaping can be disabled"""
    # Set max_nbytes to None to disable the auto memmaping feature
    p = MemmapingPool(3, max_nbytes=None, temp_folder=tmpdir.strpath)
    try:

        # Check that the tempfolder is empty
        assert os.listdir(tmpdir.strpath) == []

        # Try with a file largish than the memmap threshold of 40 bytes
        large = np.ones(100, dtype=np.float64)
        assert large.nbytes == 800
        p.map(check_array, [(large, i, 1.0) for i in range(large.shape[0])])

        # Check that the tempfolder is still empty
        assert os.listdir(tmpdir.strpath) == []

    finally:
        # Cleanup open file descriptors
        p.terminate()
        del p

# 需要导入模块: from joblib.pool import MemmapingPool [as 别名]
# 或者: from joblib.pool.MemmapingPool import terminate [as 别名]
def test_workaround_against_bad_memmap_with_copied_buffers(tmpdir):
    """Check that memmaps with a bad buffer are returned as regular arrays

    Unary operations and ufuncs on memmap instances return a new memmap
    instance with an in-memory buffer (probably a numpy bug).
    """
    assert_array_equal = np.testing.assert_array_equal

    p = MemmapingPool(3, max_nbytes=10, temp_folder=tmpdir.strpath)
    try:
        # Send a complex, large-ish view on a array that will be converted to
        # a memmap in the worker process
        a = np.asarray(np.arange(6000).reshape((1000, 2, 3)),
                       order='F')[:, :1, :]

        # Call a non-inplace multiply operation on the worker and memmap and
        # send it back to the parent.
        b = p.apply_async(_worker_multiply, args=(a, 3)).get()
        assert not has_shareable_memory(b)
        assert_array_equal(b, 3 * a)
    finally:
        p.terminate()
        del p