Python numpy.memmap函数代码示例

def convert(in_name, out_name):
    """convert the file identified by filename in_name to a complex numpy array and store it to a file named out_name"""
    wav = wave.open(in_name,'rb')
    verifyfileformat(wav)

    length = wav.getnframes()
    channels = wav.getnchannels()

    logging.info('length: {} frames, channels: {}'.format(length, channels))
    wav.close()
   
    # now that we know the format is valid, access data directly
    npinfile = np.memmap(in_name, dtype=np.int16, mode='r', offset=44) 
    if npinfile.shape[0]/2 != length:
        raise TypeError('frame mismatch in direct access')

    # our output file, this will be an npy binary holding complex64 types
    npfile = np.memmap(out_name, dtype=np.complex64,
                       mode='w+',
                       shape=(length,))

    # convert input to complex output
    npfile[:] = npinfile[0::2] + 1j * npinfile[1::2]
    
    # cleanup
    del npinfile
    del npfile

def read_ply(ply_filename):
    vfile = tempfile.mktemp()
    ffile = tempfile.mktemp()
    reader = ply_reader.PlyReader(ply_filename)
    
    v_id = 0
    f_id = 0

    # Reading the header
    for evt, data in reader.read():
        if evt == ply_reader.EVENT_HEADER:
            n_vertices, n_faces = data
            vertices = np.memmap(vfile, dtype='float64', shape = (n_vertices,3),
                                mode='w+')
            faces = np.memmap(ffile, dtype='int64', shape = (n_faces,3),
                              mode='w+')
            break

    # Reading the vertices and faces
    for evt, data in reader.read():
        if evt == ply_reader.EVENT_VERTEX:
            current_vertex = data
            vertices[v_id] = current_vertex
            v_id += 1

        elif evt == ply_reader.EVENT_FACE:
            faces[f_id] = data
            f_id += 1

    return vertices, faces

def test_score_memmap():
    # Ensure a scalar score of memmap type is accepted
    iris = load_iris()
    X, y = iris.data, iris.target
    clf = MockClassifier()
    tf = tempfile.NamedTemporaryFile(mode='wb', delete=False)
    tf.write(b'Hello world!!!!!')
    tf.close()
    scores = np.memmap(tf.name, dtype=np.float64)
    score = np.memmap(tf.name, shape=(), mode='r', dtype=np.float64)
    try:
        cross_val_score(clf, X, y, scoring=lambda est, X, y: score)
        # non-scalar should still fail
        assert_raises(ValueError, cross_val_score, clf, X, y,
                      scoring=lambda est, X, y: scores)
    finally:
        # Best effort to release the mmap file handles before deleting the
        # backing file under Windows
        scores, score = None, None
        for _ in range(3):
            try:
                os.unlink(tf.name)
                break
            except WindowsError:
                sleep(1.)

 def sim_calc(self):
     nt = self.corpora[0]
     self.scores = {}
     for corp in self.corpora:
         i_nt = []
         i_c2 = []
         rows = self.ekk_rows[corp[0]]
         for i, word in enumerate(self.ekk_rows['NT']):
             if word in rows:
                 i_nt.append(i)
                 i_c2.append(self.ekk_rows[corp[0]].index(word))
         d_c2 = np.memmap(
             '{0}{1}/{4}/{2}/{5}_{2}_lems=False_{4}_min_occ={3}_{6}no_stops=False_NORMED.dat'.format(
                 self.base, corp[0], corp[1], corp[2], self.english, self.prefix, self.svd),
             dtype='float32', shape=(len(rows), len(rows)))[i_c2]
         d_c2 = d_c2[:, i_c2]
         d_nt = np.memmap(
             '{0}{1}/{4}/{2}/{5}_{2}_lems=False_{4}_min_occ={3}_{6}no_stops=False_NORMED.dat'.format(
                 self.base, nt[0], nt[1], nt[2], self.english, self.prefix,
                 self.svd), dtype='float32',
             shape=(len(self.ekk_rows['NT']), len(self.ekk_rows['NT'])))[
             i_nt]
         d_nt = d_nt[:, i_nt]
         self.scores['{0}_{1}'.format('NT', corp[0])] = np.average(np.diag(
             1 - pairwise_distances(d_nt, d_c2, metric='cosine',
                                    n_jobs=12)))

 def save(self, dirname = None):
     """Save the current rdfspace to a directory (by default the directory in which indexes are stored)"""
     if dirname is None and self._index_dir is not None:
         dirname = self._index_dir
     if not os.path.exists(dirname):
         os.makedirs(dirname)
     # We memmap big matrices, as pickle eats the whole RAM
     # We don't save the full adjacency matrix
     ut_m = np.memmap(os.path.join(dirname, 'ut.dat'), dtype='float64', mode='w+', shape=self._ut_shape)
     ut_m[:] = self._ut[:]
     s_m = np.memmap(os.path.join(dirname, 's.dat'), dtype='float64', mode='w+', shape=self._s_shape)
     s_m[:] = self._s[:]
     vt_m = np.memmap(os.path.join(dirname, 'vt.dat'), dtype='float64', mode='w+', shape=self._vt_shape)
     vt_m[:] = self._vt[:]
     if self._index_dir is None:
         # The index is in memory, we'll pickle it with the rest
         (adjacency, ut, s, vt) = (self._adjacency, self._ut, self._s, self._vt)
         (self._adjacency, self._ut, self._s, self._vt) = (None, None, None, None)
         f = open(os.path.join(dirname, 'space.dat'), 'w')
         pickle.dump(self, f)
         f.close()
         (self._adjacency, self._ut, self._s, self._vt) = (adjacency, ut, s, vt)
     else:
         # Flushing indexes
         self._uri_index.close()
         self._index_uri.close()
         # The index is stored in dbm, we will exclude it from the pickle
         (adjacency, ut, s, vt) = (self._adjacency, self._ut, self._s, self._vt)
         (self._adjacency, self._ut, self._s, self._vt, self._uri_index, self._index_uri) = (None, None, None, None, None, None)
         f = open(os.path.join(dirname, 'space.dat'), 'w')
         pickle.dump(self, f)
         f.close()
         (self._adjacency, self._ut, self._s, self._vt) = (adjacency, ut, s, vt)
         self._uri_index = dbm.open(os.path.join(dirname, 'uri_index'), 'r')
         self._index_uri = dbm.open(os.path.join(dirname, 'index_uri'), 'r')

def memmap(docompute, dowrite, verbose):

    afilename = os.path.join(OUT_DIR, "memmap-a.bin")
    bfilename = os.path.join(OUT_DIR, "memmap-b.bin")
    rfilename = os.path.join(OUT_DIR, "memmap-output.bin")
    if dowrite:
        t0 = time()
        a = np.memmap(afilename, dtype='float32', mode='w+', shape=shape)
        b = np.memmap(bfilename, dtype='float32', mode='w+', shape=shape)

        # Fill arrays a and b
        #row = np.linspace(0, 1, ncols)
        row = np.arange(0, ncols, dtype='float32')
        for i in range(nrows):
            a[i] = row * (i + 1)
            b[i] = row * (i + 1) * 2
        del a, b  # flush data
        print("[numpy.memmap] Time for creating inputs:",
              round(time() - t0, 3))

    if docompute:
        t0 = time()
        # Reopen inputs in read-only mode
        a = np.memmap(afilename, dtype='float32', mode='r', shape=shape)
        b = np.memmap(bfilename, dtype='float32', mode='r', shape=shape)
        # Create the array output
        r = np.memmap(rfilename, dtype='float32', mode='w+', shape=shape)
        # Do the computation row by row
        for i in range(nrows):
            r[i] = eval(expr, {'a': a[i], 'b': b[i]})
        if verbose:
            print("First ten values:", r[0, :10])
        del a, b
        del r  # flush output data
        print("[numpy.memmap] Time for compute & save:", round(time() - t0, 3))

    def parse_graph(self, graph_path, data_dir='data', load_edges=False, extend_paths=2):
        graph = parser.Graph(graph_path)
        self.from_nodes, self.to_nodes = graph.get_mappings()
        graph.save_mappings(self.output_dir)

        if load_edges:
            self.inverse_degrees = np.memmap(
                os.path.join(data_dir, 'inverse_degrees.mat'),
                mode='r',
                dtype='float32'
            )
            self.from_to_idxs = np.memmap(
                os.path.join(data_dir, 'from_to.mat'),
                mode='r',
                dtype='int32'
            )
            self.from_to_idxs = np.reshape(self.from_to_idxs, newshape=(self.inverse_degrees.shape[0], 2))
        else:
            from_to_idxs, inverse_degrees = graph.extend_graph(max_degree=extend_paths)
            self.from_to_idxs = np.memmap(
                os.path.join(data_dir, 'from_to.mat'),
                mode='r+',
                shape=from_to_idxs.shape,
                dtype='int32'
            )
            self.from_to_idxs[:] = from_to_idxs[:]
            self.inverse_degrees = np.memmap(
                os.path.join(data_dir, 'inverse_degrees.mat'),
                mode='r+',
                shape=inverse_degrees.shape,
                dtype='float32'
            )
            self.inverse_degrees[:] = inverse_degrees[:]

	def _train(self, x):
# 		print self.dtype
		if len(x) > self.defaultOutputLength:
			self.defaultOutputLength = len(x)
		self.cacheLength += len(x)
		if self.cache is None:
			if self.cacheSize == -1:
				#self.cache = np.memmap(self.cacheName, dtype='float32', mode='w+', shape = x.shape)
				self.cache = np.memmap(self.cacheName, dtype=self.dtype, mode='w+', shape = x.shape)
			else:
				#self.cache = np.memmap(self.cacheName, dtype='float32', mode='w+', shape = (self.cacheSize, len(x[0])))
				self.cache = np.memmap(self.cacheName, dtype=self.dtype, mode='w+', shape = (self.cacheSize, len(x[0])))
		elif self.cacheSize == -1:
			self.reshape((self.cache.shape[0]+len(x), len(x[0])))
# 			print x[0][0].dtype.itemsize
# 			print self.cache._mmap.size()
# 			#self.cache._mmap.resize( (self.cache.shape[0]+len(x), len(x[0])) )
# 			print self.cache.shape
# 			newShape = (self.cache.shape[0]+len(x), len(x[0]))
# 			memmap_resize( newShape, self.cache )
# 			del self.cache
# 			self.cache = np.memmap(self.cacheName, dtype=self.dtype, mode='w+', shape = newShape)
# 			print "new size: "+str(self.cache._mmap.size())
# 			print self.cache.reshape(newShape)
		self.cache[self.cachePos:self.cachePos+len(x)] = x
# 		print self.cache._mmap.size()
# 		print self.cache[0][0]
# 		print self.cache[0][0].dtype.itemsize
# 		print "---"
		self.cachePos += len(x)

def main(A):
    """convolve the tau(mass) field, 
    add in thermal broadening and redshift distortion """

    sightlines = Sightlines(A)
    maker = SpectraMaker(A, sightlines)
    fgpa = FGPAmodel(A)

    Npixels = sightlines.Npixels.sum()

    spectaureal = numpy.memmap(A.SpectraOutputTauReal, mode='w+', 
            dtype='f4', shape=Npixels)
    spectaured = numpy.memmap(A.SpectraOutputTauRed, mode='w+', 
            dtype='f4', shape=Npixels)
    specdelta = numpy.memmap(A.SpectraOutputDelta, mode='w+', 
            dtype='f4', shape=Npixels)

    def work(i):
        sl2 = slice(sightlines.PixelOffset[i], 
                sightlines.PixelOffset[i] + sightlines.Npixels[i])
        result =  maker.convolve(i, Afunc=fgpa.Afunc, Bfunc=fgpa.Bfunc)
        spectaureal[sl2] = result.taureal
        spectaured[sl2] = result.taured
        specdelta[sl2] = result.delta
        sightlines.Z_RED[i] = result.Zqso
    chunkmap(work, range(len(sightlines)), 100)

    spectaureal.flush()
    spectaured.flush()
    specdelta.flush()
    sightlines.Z_RED.flush()

    def extract_to_memmap(self):
        """
        Allocate a memmap, fill it with extracted features, return r/o view.
        """
        filename = self.filename
        feature_shp = self.feature_shp
        print('Creating memmap %s for features of shape %s' % (
                                              filename,
                                              str(feature_shp)))
        features_fp = np.memmap(filename,
            dtype='float32',
            mode='w+',
            shape=feature_shp)
        info = open(filename+'.info', 'w')
        cPickle.dump(('float32', feature_shp), info)
        del info

        self.extract_to_storage(features_fp)

        # -- docs here:
        #    http://docs.scipy.org/doc/numpy/reference/generated/numpy.memmap.html
        #    say that deletion is the way to flush changes !?
        del features_fp
        rval = np.memmap(self.filename,
            dtype='float32',
            mode='r',
            shape=feature_shp)
        return rval

    def next(self):
        # for python 2.x
        # Keep under lock only the mechainsem which advance the indexing of each batch
        # see # http://anandology.com/blog/using-iterators-and-generators/
        with self.lock:
            song_idx, self.cur_song = self.cur_song, self.cur_song+1

        bX, bY = (None, None)
        if song_idx < self.n_songs:
            x_path = self.data[self.sidstr[song_idx]]['X_path']
            y_path = self.data[self.sidstr[song_idx]]['y_path']
            bX = np.memmap(
                x_path,
                dtype='float32',
                mode='r',
                shape=tuple(self.data[self.sidstr[song_idx]]['X_shape'])
                )
            bY = np.memmap(
                y_path,
                dtype='float32',
                mode='r',
                shape=tuple(self.data[self.sidstr[song_idx]]['y_shape'])
                )
            return bX, bY
        else:
            raise StopIteration()
        return bX, bY

    def __init__(self,fdata,fndata):
        #dump to binary
        print('Initialize binary from ' + fdata)

        if (not os.path.isfile(fndata)):

            print('create Epix100a flat file ' + fndata + ' from ' + fdata)
            
            datain=Epix100a(fdata);

            #write header
            binheader=np.zeros(16).astype(np.uint32);
            binheader[0:6]=[datain.nframes, datain.my*datain.mx, datain.my, datain.mx, datain.nblocks, datain.nbcols];
            binheader.tofile(fndata);    

            #write data
            dataout=np.memmap(fndata,dtype=np.int16,mode='r+', shape=(datain.nframes,datain.my,datain.mx),offset=64);
            t0=time.clock();
            for iframe in range(datain.nframes):
                dataout[iframe]=datain.frame(iframe);
                if (iframe%100==0):
                    #progress(iframe,nframes,iframe);
                    print (str(iframe)+' - '+str(1000*(time.clock()-t0)/(iframe+1))+' ms. average frame: '+str(np.mean(datain.frame(iframe))))
            dataout.flush();
            
            del dataout;
            del datain;
        
        #get nr of frames
        else:
            print(fndata + ' file already exists.')
        data=np.memmap(fndata,dtype=np.uint32,mode='r',shape=((64)),offset=0); 
        self.nframes=data[0]; self.nframesize=data[1]; self.my=data[2]; self.mx=data[3]; self.nblocks=data[4]; self.nbcols=data[5];
        self.data=np.memmap(fndata,dtype=np.int16,mode='c',shape=(self.nframes,self.my,self.mx),offset=64);

def get_sequence(mraw_path, file_shape, nmax=None, offset=0):
    '''
    Get a sequence of image files as 3D numpy array.

    :param mraw_path: path to .mraw file containing image data
    :param file_shape: tuple, (ntotal, height, width) of images in .mraw file
    :param nmax: maximum number of images in sequence
    :param offset: First image to be read
    :return: 3D array of image sequence
    '''
    ntotal, h, w = file_shape
    byte_size = 2*h*w                   # Number of bytes for one image 
    byte_offset = offset * byte_size    # Offset to first byte to be read

    # If only a single image was requested:
    if nmax and nmax == 1:
        with open(mraw_path, 'rb') as mraw:
            imarray = np.memmap(mraw, dtype=np.uint16, offset=byte_offset, mode='r', shape=(h, w))
    # Only display nmax or less images:
    elif nmax and ntotal > nmax:
        image_step = ntotal//nmax
        with open(mraw_path, 'rb') as mraw:
            memmap = np.memmap(mraw, dtype=np.uint16, offset=byte_offset, mode='r', shape=(ntotal-offset, h, w))
            imarray = memmap[::image_step, :, :]
    # If there are less than nmax images:
    else:
        with open(mraw_path, 'rb') as mraw:
            imarray = np.memmap(mraw, dtype=np.uint16, offset=byte_offset, mode='r', shape=(ntotal-offset, h, w))

    return imarray

    def __next__(self):
        #check to see if at end of chunks
        if self._chunk_counter==self.num_chunks:
            offset = int(self._chunk_counter * self.chunksize)
            row_size = self.rmndr_row_size
            self._chunk_counter += 1
        elif self._chunk_counter < self.num_chunks:
            offset = int(self._chunk_counter * self.chunksize)
            end_dp = (self._chunk_counter+1) + self.chunksize
            row_size = self.chunk_row_size
            self._chunk_counter += 1
        elif self._chunk_counter > self.num_chunks:
            raise StopIteration

        if self.abr.header['f_structure']['nDataFormat'][0]==1: #float data
            data = memmap(self.abr.fid, dtype = float32, shape = (row_size,self.ncols), offset = offset+self.offset_base)
            return data

        elif self.abr.header['f_structure']['nDataFormat'][0]==0: #integer data
            try:
                data = memmap(self.abr.fid, dtype = int16, shape = (row_size,self.ncols),
                              mode = 'r',offset = offset + self.offset_base)
            except ValueError:
                pdb.set_trace()
            data = data[:].astype(float32)
            data = self.abr.scale_int_data(data)
            return data

    def update(self):
        """ Updates L-BFGS algorithm history
        """
        unix.cd(self.path)

        s = self.load('m_new') - self.load('m_old')
        y = self.load('g_new') - self.load('g_old')

        m = len(s)
        n = self.memory

        if self.memory_used == 0:
            S = np.memmap('LBFGS/S', mode='w+', dtype='float32', shape=(m, n))
            Y = np.memmap('LBFGS/Y', mode='w+', dtype='float32', shape=(m, n))
            S[:, 0] = s
            Y[:, 0] = y
            self.memory_used = 1

        else:
            S = np.memmap('LBFGS/S', mode='r+', dtype='float32', shape=(m, n))
            Y = np.memmap('LBFGS/Y', mode='r+', dtype='float32', shape=(m, n))
            S[:, 1:] = S[:, :-1]
            Y[:, 1:] = Y[:, :-1]
            S[:, 0] = s
            Y[:, 0] = y

            if self.memory_used < self.memory:
                self.memory_used += 1

        return S, Y