Python format.open_memmap方法代码示例

# 需要导入模块: from numpy.lib import format [as 别名]
# 或者: from numpy.lib.format import open_memmap [as 别名]
def test_version_2_0_memmap():
    # requires more than 2 byte for header
    dt = [(("%d" % i) * 100, float) for i in range(500)]
    d = np.ones(1000, dtype=dt)
    tf = tempfile.mktemp('', 'mmap', dir=tempdir)

    # 1.0 requested but data cannot be saved this way
    assert_raises(ValueError, format.open_memmap, tf, mode='w+', dtype=d.dtype,
                            shape=d.shape, version=(1, 0))

    ma = format.open_memmap(tf, mode='w+', dtype=d.dtype,
                            shape=d.shape, version=(2, 0))
    ma[...] = d
    del ma

    with warnings.catch_warnings(record=True) as w:
        warnings.filterwarnings('always', '', UserWarning)
        ma = format.open_memmap(tf, mode='w+', dtype=d.dtype,
                                shape=d.shape, version=None)
        assert_(w[0].category is UserWarning)
        ma[...] = d
        del ma

    ma = format.open_memmap(tf, mode='r')
    assert_array_equal(ma, d) 

# 需要导入模块: from numpy.lib import format [as 别名]
# 或者: from numpy.lib.format import open_memmap [as 别名]
def load_samples(data_folder, mode='r'):
    """Load sampled results as a dictionary of numpy memmap.

    Args:
        data_folder (str): the folder from which to use the samples
        mode (str): the mode in which to open the memory mapped sample files (see numpy mode parameter)

    Returns:
        dict: the memory loaded samples per sampled parameter.
    """
    data_dict = {}
    for fname in glob.glob(os.path.join(data_folder, '*.samples.npy')):
        samples = open_memmap(fname, mode=mode)
        map_name = os.path.basename(fname)[0:-len('.samples.npy')]
        data_dict.update({map_name: samples})
    return data_dict 

# 需要导入模块: from numpy.lib import format [as 别名]
# 或者: from numpy.lib.format import open_memmap [as 别名]
def load_sample(fname, mode='r'):
    """Load an matrix of samples from a ``.samples.npy`` file.

    This will open the samples as a numpy memory mapped array.

    Args:
        fname (str): the name of the file to load, suffix of ``.samples.npy`` is not required.
        mode (str): the mode in which to open the memory mapped sample files (see numpy mode parameter)

    Returns:
        ndarray: a memory mapped array with the results
    """
    if not os.path.isfile(fname) and not os.path.isfile(fname + '.samples.npy'):
        raise ValueError('Could not find sample results at the location "{}"'.format(fname))

    if not os.path.isfile(fname):
        fname += '.samples.npy'

    return open_memmap(fname, mode=mode) 

# 需要导入模块: from numpy.lib import format [as 别名]
# 或者: from numpy.lib.format import open_memmap [as 别名]
def _store_sample(self, optimization_results, roi_indices, sample_ind):
        """Store the optimization results as a next sample."""
        if not os.path.exists(self._output_dir):
            os.makedirs(self._output_dir)

        if self._sample_storage is None:
            self._sample_storage = {}
            for key, value in optimization_results.items():
                samples_path = os.path.join(self._output_dir, key + '.samples.npy')
                mode = 'w+'

                if os.path.isfile(samples_path):
                    mode = 'r+'
                    current_results = open_memmap(samples_path, mode='r')
                    if current_results.shape[1] != self._nmr_samples:
                        mode = 'w+'  # opening the memmap with w+ creates a new one
                    del current_results  # closes the memmap

                shape = [self._total_nmr_voxels, self._nmr_samples]
                if value.ndim > 1:
                    shape.extend(value.shape[1:])
                self._sample_storage[key] = open_memmap(samples_path, mode=mode, dtype=value.dtype, shape=tuple(shape))

        for key, value in optimization_results.items():
            self._sample_storage[key][roi_indices, sample_ind] = value 

# 需要导入模块: from numpy.lib import format [as 别名]
# 或者: from numpy.lib.format import open_memmap [as 别名]
def combine(self):
        super().combine()

        statistic_maps = {}
        for name in self._sample_storage:
            samples_path = os.path.join(self._output_dir, name + '.samples.npy')
            samples = open_memmap(samples_path, mode='r')
            statistic_maps[name] = np.mean(samples, axis=1)
            statistic_maps[name + '.std'] = np.std(samples, axis=1)

        write_all_as_nifti(restore_volumes(statistic_maps, self._mask),
                           os.path.join(self._output_dir, 'univariate_normal'),
                           nifti_header=self._nifti_header,
                           gzip=self._write_volumes_gzipped)

        write_all_as_nifti({'UsedMask': self._mask}, self._output_dir, nifti_header=self._nifti_header,
                           gzip=self._write_volumes_gzipped)

        if not self._keep_samples:
            for ind, name in enumerate(self._model.get_free_param_names()):
                os.remove(os.path.join(self._output_dir, name + '.samples.npy'))
        else:
            return load_samples(self._output_dir) 

# 需要导入模块: from numpy.lib import format [as 别名]
# 或者: from numpy.lib.format import open_memmap [as 别名]
def _create_schema(self, *, remote_operation: bool = False):
        """stores the shape and dtype as the schema of a column.

        Parameters
        ----------
        remote_operation : optional, kwarg only, bool
            if this schema is being created from a remote fetch operation, then do not
            place the file symlink in the staging directory. Instead symlink it
            to a special remote staging directory. (default is False, which places the
            symlink in the stage data directory.)
        """
        uid = random_string()
        file_path = self.DATADIR.joinpath(f'{uid}.npy')
        m = open_memmap(file_path,
                        mode='w+',
                        dtype=self.schema_dtype,
                        shape=(COLLECTION_SIZE, *self.schema_shape))
        self.wFp[uid] = m
        self.w_uid = uid
        self.hIdx = 0

        process_dir = self.REMOTEDIR if remote_operation else self.STAGEDIR
        Path(process_dir, f'{uid}.npy').touch() 

# 需要导入模块: from numpy.lib import format [as 别名]
# 或者: from numpy.lib.format import open_memmap [as 别名]
def test_memmap_roundtrip():
    # Fixme: used to crash on windows
    if not (sys.platform == 'win32' or sys.platform == 'cygwin'):
        for arr in basic_arrays + record_arrays:
            if arr.dtype.hasobject:
                # Skip these since they can't be mmap'ed.
                continue
            # Write it out normally and through mmap.
            nfn = os.path.join(tempdir, 'normal.npy')
            mfn = os.path.join(tempdir, 'memmap.npy')
            fp = open(nfn, 'wb')
            try:
                format.write_array(fp, arr)
            finally:
                fp.close()

            fortran_order = (
                arr.flags.f_contiguous and not arr.flags.c_contiguous)
            ma = format.open_memmap(mfn, mode='w+', dtype=arr.dtype,
                                    shape=arr.shape, fortran_order=fortran_order)
            ma[...] = arr
            del ma

            # Check that both of these files' contents are the same.
            fp = open(nfn, 'rb')
            normal_bytes = fp.read()
            fp.close()
            fp = open(mfn, 'rb')
            memmap_bytes = fp.read()
            fp.close()
            assert_equal_(normal_bytes, memmap_bytes)

            # Check that reading the file using memmap works.
            ma = format.open_memmap(nfn, mode='r')
            del ma 

# 需要导入模块: from numpy.lib import format [as 别名]
# 或者: from numpy.lib.format import open_memmap [as 别名]
def test_memmap_roundtrip():
    # Fixme: test crashes nose on windows.
    if not (sys.platform == 'win32' or sys.platform == 'cygwin'):
        for arr in basic_arrays + record_arrays:
            if arr.dtype.hasobject:
                # Skip these since they can't be mmap'ed.
                continue
            # Write it out normally and through mmap.
            nfn = os.path.join(tempdir, 'normal.npy')
            mfn = os.path.join(tempdir, 'memmap.npy')
            fp = open(nfn, 'wb')
            try:
                format.write_array(fp, arr)
            finally:
                fp.close()

            fortran_order = (
                arr.flags.f_contiguous and not arr.flags.c_contiguous)
            ma = format.open_memmap(mfn, mode='w+', dtype=arr.dtype,
                                    shape=arr.shape, fortran_order=fortran_order)
            ma[...] = arr
            del ma

            # Check that both of these files' contents are the same.
            fp = open(nfn, 'rb')
            normal_bytes = fp.read()
            fp.close()
            fp = open(mfn, 'rb')
            memmap_bytes = fp.read()
            fp.close()
            yield assert_equal_, normal_bytes, memmap_bytes

            # Check that reading the file using memmap works.
            ma = format.open_memmap(nfn, mode='r')
            del ma 

# 需要导入模块: from numpy.lib import format [as 别名]
# 或者: from numpy.lib.format import open_memmap [as 别名]
def test_memmap_roundtrip():
    # XXX: test crashes nose on windows. Fix this
    if not (sys.platform == 'win32' or sys.platform == 'cygwin'):
        for arr in basic_arrays + record_arrays:
            if arr.dtype.hasobject:
                # Skip these since they can't be mmap'ed.
                continue
            # Write it out normally and through mmap.
            nfn = os.path.join(tempdir, 'normal.npy')
            mfn = os.path.join(tempdir, 'memmap.npy')
            fp = open(nfn, 'wb')
            try:
                format.write_array(fp, arr)
            finally:
                fp.close()

            fortran_order = (arr.flags.f_contiguous and not arr.flags.c_contiguous)
            ma = format.open_memmap(mfn, mode='w+', dtype=arr.dtype,
                shape=arr.shape, fortran_order=fortran_order)
            ma[...] = arr
            del ma

            # Check that both of these files' contents are the same.
            fp = open(nfn, 'rb')
            normal_bytes = fp.read()
            fp.close()
            fp = open(mfn, 'rb')
            memmap_bytes = fp.read()
            fp.close()
            yield assert_equal, normal_bytes, memmap_bytes

            # Check that reading the file using memmap works.
            ma = format.open_memmap(nfn, mode='r')
            #yield assert_array_equal, ma, arr
            del ma 

# 需要导入模块: from numpy.lib import format [as 别名]
# 或者: from numpy.lib.format import open_memmap [as 别名]
def _write_sample_results(self, results, roi_indices):
        """Write the sample results to a .npy file.

        If the given sample files do not exists or if the existing file is not large enough it will create one
        with enough storage to hold all the samples for the given total_nmr_voxels.
        On storing it should also be given a list of voxel indices with the indices of the voxels that are being stored.

        Args:
            results (dict): the samples to write
            roi_indices (ndarray): the roi indices of the voxels we computed
        """
        if not os.path.exists(self._output_dir):
            os.makedirs(self._output_dir)

        for fname in os.listdir(self._output_dir):
            if fname.endswith('.samples.npy'):
                chain_name = fname[0:-len('.samples.npy')]
                if chain_name not in results:
                    os.remove(os.path.join(self._output_dir, fname))

        for output_name, samples in results.items():
            save_indices = self._samples_to_save_method.indices_to_store(output_name, samples.shape[1])
            samples_path = os.path.join(self._output_dir, output_name + '.samples.npy')
            mode = 'w+'

            if os.path.isfile(samples_path):
                mode = 'r+'
                current_results = open_memmap(samples_path, mode='r')
                if current_results.shape[1] != len(save_indices):
                    mode = 'w+'
                del current_results  # closes the memmap

            saved = open_memmap(samples_path, mode=mode, dtype=samples.dtype,
                                shape=(self._total_nmr_voxels, len(save_indices)))
            saved[roi_indices, :] = samples[:, save_indices]
            del saved