Python util.filechunkiter函数代码示例

def snapshot_wdir(ui, repo, files, tmproot):
    '''snapshot files from working directory.
    if not using snapshot, -I/-X does not work and recursive diff
    in tools like kdiff3 and meld displays too many files.'''
    repo_root = repo.root

    dirname = os.path.basename(repo_root)
    if dirname == "":
        dirname = "root"
    base = os.path.join(tmproot, dirname)
    os.mkdir(base)
    ui.note(_('making snapshot of %d files from working dir\n') %
            (len(files)))

    fns_and_mtime = []

    for fn in files:
        wfn = util.pconvert(fn)
        ui.note('  %s\n' % wfn)
        dest = os.path.join(base, wfn)
        destdir = os.path.dirname(dest)
        if not os.path.isdir(destdir):
            os.makedirs(destdir)

        fp = open(dest, 'wb')
        for chunk in util.filechunkiter(repo.wopener(wfn)):
            fp.write(chunk)
        fp.close()

        fns_and_mtime.append((dest, os.path.join(repo_root, fn),
            os.path.getmtime(dest)))


    return dirname, fns_and_mtime

 def drain(self):
     '''
     Need to read all data from request, httplib is half-duplex
     '''
     length = int(self.env.get('CONTENT_LENGTH', 0))
     for s in util.filechunkiter(self.inp, limit=length):
         pass

def hexsha1(data):
    """hexsha1 returns the hex-encoded sha1 sum of the data in the file-like
    object data"""
    h = util.sha1()
    for chunk in util.filechunkiter(data):
        h.update(chunk)
    return h.hexdigest()

def consumestreamclone(repo, fp):
    """Apply the contents from a streaming clone file.

    This code is copied from Mercurial. Until Mercurial 3.5, this code was
    a closure in wireproto.py and not consumeable by extensions.
    """
    lock = repo.lock()
    try:
        repo.ui.status(_('streaming all changes\n'))
        l = fp.readline()
        try:
            total_files, total_bytes = map(int, l.split(' ', 1))
        except (ValueError, TypeError):
            raise error.ResponseError(
                _('unexpected response from remote server:'), l)
        repo.ui.status(_('%d files to transfer, %s of data\n') %
                       (total_files, util.bytecount(total_bytes)))
        handled_bytes = 0
        repo.ui.progress(_('clone'), 0, total=total_bytes)
        start = time.time()

        tr = repo.transaction(_('clone'))
        try:
            for i in xrange(total_files):
                # XXX doesn't support '\n' or '\r' in filenames
                l = fp.readline()
                try:
                    name, size = l.split('\0', 1)
                    size = int(size)
                except (ValueError, TypeError):
                    raise error.ResponseError(
                        _('unexpected response from remote server:'), l)
                if repo.ui.debugflag:
                    repo.ui.debug('adding %s (%s)\n' %
                                  (name, util.bytecount(size)))
                # for backwards compat, name was partially encoded
                ofp = repo.svfs(store.decodedir(name), 'w')
                for chunk in util.filechunkiter(fp, limit=size):
                    handled_bytes += len(chunk)
                    repo.ui.progress(_('clone'), handled_bytes,
                                     total=total_bytes)
                    ofp.write(chunk)
                ofp.close()
            tr.close()
        finally:
            tr.release()

        # Writing straight to files circumvented the inmemory caches
        repo.invalidate()

        elapsed = time.time() - start
        if elapsed <= 0:
            elapsed = 0.001
        repo.ui.progress(_('clone'), None)
        repo.ui.status(_('transferred %s in %.1f seconds (%s/sec)\n') %
                       (util.bytecount(total_bytes), elapsed,
                        util.bytecount(total_bytes / elapsed)))
    finally:
        lock.release()

 def _getfile(self, tmpfile, filename, hash):
     path = lfutil.findfile(self.remote, hash)
     if not path:
         raise basestore.StoreError(filename, hash, self.url,
             _("can't get file locally"))
     with open(path, 'rb') as fd:
         return lfutil.copyandhash(
             util.filechunkiter(fd), tmpfile)

def hashfile(file):
    if not os.path.exists(file):
        return ''
    hasher = hashlib.sha1('')
    with open(file, 'rb') as fd:
        for data in util.filechunkiter(fd):
            hasher.update(data)
    return hasher.hexdigest()

def hashfile(file):
    if not os.path.exists(file):
        return ''
    hasher = util.sha1('')
    fd = open(file, 'rb')
    for data in util.filechunkiter(fd, 128 * 1024):
        hasher.update(data)
    fd.close()
    return hasher.hexdigest()

def link(src, dest):
    try:
        util.oslink(src, dest)
    except OSError:
        # if hardlinks fail, fallback on atomic copy
        dst = util.atomictempfile(dest)
        for chunk in util.filechunkiter(open(src, 'rb')):
            dst.write(chunk)
        dst.close()
        os.chmod(dest, os.stat(src).st_mode)

def copytostoreabsolute(repo, file, hash):
    util.makedirs(os.path.dirname(storepath(repo, hash)))
    if inusercache(repo.ui, hash):
        link(usercachepath(repo.ui, hash), storepath(repo, hash))
    else:
        dst = util.atomictempfile(storepath(repo, hash))
        for chunk in util.filechunkiter(open(file, 'rb')):
            dst.write(chunk)
        dst.close()
        util.copymode(file, storepath(repo, hash))
        linktousercache(repo, hash)

def consumev1(repo, fp, filecount, bytecount):
    """Apply the contents from version 1 of a streaming clone file handle.

    This takes the output from "streamout" and applies it to the specified
    repository.

    Like "streamout," the status line added by the wire protocol is not handled
    by this function.
    """
    lock = repo.lock()
    try:
        repo.ui.status(_('%d files to transfer, %s of data\n') %
                       (filecount, util.bytecount(bytecount)))
        handled_bytes = 0
        repo.ui.progress(_('clone'), 0, total=bytecount)
        start = time.time()

        tr = repo.transaction(_('clone'))
        try:
            for i in xrange(filecount):
                # XXX doesn't support '\n' or '\r' in filenames
                l = fp.readline()
                try:
                    name, size = l.split('\0', 1)
                    size = int(size)
                except (ValueError, TypeError):
                    raise error.ResponseError(
                        _('unexpected response from remote server:'), l)
                if repo.ui.debugflag:
                    repo.ui.debug('adding %s (%s)\n' %
                                  (name, util.bytecount(size)))
                # for backwards compat, name was partially encoded
                ofp = repo.svfs(store.decodedir(name), 'w')
                for chunk in util.filechunkiter(fp, limit=size):
                    handled_bytes += len(chunk)
                    repo.ui.progress(_('clone'), handled_bytes, total=bytecount)
                    ofp.write(chunk)
                ofp.close()
            tr.close()
        finally:
            tr.release()

        # Writing straight to files circumvented the inmemory caches
        repo.invalidate()

        elapsed = time.time() - start
        if elapsed <= 0:
            elapsed = 0.001
        repo.ui.progress(_('clone'), None)
        repo.ui.status(_('transferred %s in %.1f seconds (%s/sec)\n') %
                       (util.bytecount(bytecount), elapsed,
                        util.bytecount(bytecount / elapsed)))
    finally:
        lock.release()

def copytostoreabsolute(repo, file, hash):
    if inusercache(repo.ui, hash):
        link(usercachepath(repo.ui, hash), storepath(repo, hash))
    elif not getattr(repo, "_isconverting", False):
        util.makedirs(os.path.dirname(storepath(repo, hash)))
        dst = util.atomictempfile(storepath(repo, hash),
                                  createmode=repo.store.createmode)
        for chunk in util.filechunkiter(open(file, 'rb')):
            dst.write(chunk)
        dst.close()
        linktousercache(repo, hash)

def copytostoreabsolute(repo, file, hash):
    if inusercache(repo.ui, hash):
        link(usercachepath(repo.ui, hash), storepath(repo, hash))
    else:
        util.makedirs(os.path.dirname(storepath(repo, hash)))
        with open(file, 'rb') as srcf:
            with util.atomictempfile(storepath(repo, hash),
                                     createmode=repo.store.createmode) as dstf:
                for chunk in util.filechunkiter(srcf):
                    dstf.write(chunk)
        linktousercache(repo, hash)

    def bail(response, headers={}):
        length = int(req.env.get('CONTENT_LENGTH', 0))
        for s in util.filechunkiter(req, limit=length):
            # drain incoming bundle, else client will not see
            # response when run outside cgi script
            pass

        status = headers.pop('status', HTTP_OK)
        req.header(headers.items())
        req.respond(status, HGTYPE)
        req.write('0\n')
        req.write(response)

def link(src, dest):
    """Try to create hardlink - if that fails, efficiently make a copy."""
    util.makedirs(os.path.dirname(dest))
    try:
        util.oslink(src, dest)
    except OSError:
        # if hardlinks fail, fallback on atomic copy
        dst = util.atomictempfile(dest)
        for chunk in util.filechunkiter(open(src, 'rb')):
            dst.write(chunk)
        dst.close()
        os.chmod(dest, os.stat(src).st_mode)

def generatestreamclone(repo):
    """Emit content for a streaming clone.

    This is a generator of raw chunks that constitute a streaming clone.

    This code is copied from Mercurial. Until Mercurial 3.5, this code was
    a closure in wireproto.py and not consumeable by extensions.
    """
    entries = []
    total_bytes = 0
    # Get consistent snapshot of repo, lock during scan.
    lock = repo.lock()
    try:
        repo.ui.debug('scanning\n')
        for name, ename, size in repo.store.walk():
            if size:
                entries.append((name, size))
                total_bytes += size
    finally:
            lock.release()

    repo.ui.debug('%d files, %d bytes to transfer\n' %
                  (len(entries), total_bytes))
    yield '%d %d\n' % (len(entries), total_bytes)

    sopener = repo.svfs
    oldaudit = sopener.mustaudit
    debugflag = repo.ui.debugflag
    sopener.mustaudit = False

    try:
        for name, size in entries:
            if debugflag:
                repo.ui.debug('sending %s (%d bytes)\n' % (name, size))
            # partially encode name over the wire for backwards compat
            yield '%s\0%d\n' % (store.encodedir(name), size)
            if size <= 65536:
                fp = sopener(name)
                try:
                    data = fp.read(size)
                finally:
                    fp.close()
                yield data
            else:
                for chunk in util.filechunkiter(sopener(name), limit=size):
                    yield chunk
    finally:
        sopener.mustaudit = oldaudit