Python utils.fallocate函数代码示例

    def writer(self, size=None):
        """
        Context manager to write a file. We create a temporary file first, and
        then return a DiskWriter object to encapsulate the state.

        :param size: optional initial size of file to explicitly allocate on
                     disk
        :raises DiskFileNoSpace: if a size is specified and allocation fails
        """
        if not os.path.exists(self.tmpdir):
            mkdirs(self.tmpdir)
        fd, tmppath = mkstemp(dir=self.tmpdir)
        try:
            if size is not None and size > 0:
                try:
                    fallocate(fd, size)
                except OSError:
                    raise DiskFileNoSpace()
            yield DiskWriter(self, fd, tmppath, self.threadpool)
        finally:
            try:
                os.close(fd)
            except OSError:
                pass
            try:
                os.unlink(tmppath)
            except OSError:
                pass

    def _preallocate(self):
        """
        The idea is to allocate space in front of an expanding db.  If it gets
        within 512k of a boundary, it allocates to the next boundary.
        Boundaries are 2m, 5m, 10m, 25m, 50m, then every 50m after.
        """
        if not DB_PREALLOCATION or self.db_file == ':memory:':
            return
        MB = (1024 * 1024)

        def prealloc_points():
            for pm in (1, 2, 5, 10, 25, 50):
                yield pm * MB
            while True:
                pm += 50
                yield pm * MB

        stat = os.stat(self.db_file)
        file_size = stat.st_size
        allocated_size = stat.st_blocks * 512
        for point in prealloc_points():
            if file_size <= point - MB / 2:
                prealloc_size = point
                break
        if allocated_size < prealloc_size:
            with open(self.db_file, 'rb+') as fp:
                fallocate(fp.fileno(), int(prealloc_size))

    def mkstemp(self, size=None):
        """
        Contextmanager to make a temporary file.

        :param size: optional initial size of file to allocate on disk
        :raises DiskFileNoSpace: if a size is specified and fallocate fails
        """
        if not os.path.exists(self.tmpdir):
            mkdirs(self.tmpdir)
        fd, self.tmppath = mkstemp(dir=self.tmpdir)
        try:
            if size is not None and size > 0:
                try:
                    fallocate(fd, size)
                except OSError:
                    raise DiskFileNoSpace()
            yield fd
        finally:
            try:
                os.close(fd)
            except OSError:
                pass
            tmppath, self.tmppath = self.tmppath, None
            try:
                os.unlink(tmppath)
            except OSError:
                pass

    def create(self, size=None):
        """
        Context manager to create a file. We create a temporary file first, and
        then return a DiskFileWriter object to encapsulate the state.

        .. note::

            An implementation is not required to perform on-disk
            preallocations even if the parameter is specified. But if it does
            and it fails, it must raise a `DiskFileNoSpace` exception.

        :param size: optional initial size of file to explicitly allocate on
                     disk
        :raises DiskFileNoSpace: if a size is specified and allocation fails
        """
        if not exists(self._tmpdir):
            mkdirs(self._tmpdir)
        fd, tmppath = mkstemp(dir=self._tmpdir)
        try:
            if size is not None and size > 0:
                try:
                    fallocate(fd, size)
                except OSError:
                    raise DiskFileNoSpace()
            yield DiskFileWriter(self._name, self._datadir, fd, tmppath,
                                 self._bytes_per_sync, self._threadpool)
        finally:
            try:
                os.close(fd)
            except OSError:
                pass
            try:
                os.unlink(tmppath)
            except OSError:
                pass

 new_delete_at = int(request.headers.get('X-Delete-At') or 0)
 if new_delete_at and new_delete_at < time.time():
     self.logger.increment('PUT.errors')
     return HTTPBadRequest(body='X-Delete-At in past', request=request,
                           content_type='text/plain')
 file = DiskFile(self.devices, device, partition, account, container,
                 obj, self.logger, disk_chunk_size=self.disk_chunk_size)
 orig_timestamp = file.metadata.get('X-Timestamp')
 upload_expiration = time.time() + self.max_upload_time
 etag = md5()
 upload_size = 0
 last_sync = 0
 with file.mkstemp() as (fd, tmppath):
     if 'content-length' in request.headers:
         try:
             fallocate(fd, int(request.headers['content-length']))
         except OSError:
             return HTTPInsufficientStorage(drive=device,
                                            request=request)
     reader = request.environ['wsgi.input'].read
     for chunk in iter(lambda: reader(self.network_chunk_size), ''):
         upload_size += len(chunk)
         if time.time() > upload_expiration:
             self.logger.increment('PUT.timeouts')
             return HTTPRequestTimeout(request=request)
         etag.update(chunk)
         while chunk:
             written = os.write(fd, chunk)
             chunk = chunk[written:]
         # For large files sync every 512MB (by default) written
         if upload_size - last_sync >= self.bytes_per_sync:

    def mkstemp(self, size=None):
        """
        Contextmanager to make a temporary file, optionally of a specified
        initial size.

        For Gluster, we first optimistically create the temporary file using
        the "rsync-friendly" .NAME.random naming. If we find that some path to
        the file does not exist, we then create that path and then create the
        temporary file again. If we get file name conflict, we'll retry using
        different random suffixes 1,000 times before giving up.
        """
        data_file = os.path.join(self.put_datadir, self._obj)

        # Assume the full directory path exists to the file already, and
        # construct the proper name for the temporary file.
        for i in range(0, 1000):
            tmpfile = '.' + self._obj + '.' + md5(self._obj +
                      str(random.random())).hexdigest()
            tmppath = os.path.join(self.put_datadir, tmpfile)
            try:
                fd = do_open(tmppath,
                             os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC)
            except GlusterFileSystemOSError as gerr:
                if gerr.errno == errno.EEXIST:
                    # Retry with a different random number.
                    continue
                if gerr.errno == errno.EIO:
                    # FIXME: Possible FUSE issue or race condition, let's
                    # sleep on it and retry the operation.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    continue
                if gerr.errno != errno.ENOENT:
                    # FIXME: Other cases we should handle?
                    raise
                if not self._obj_path:
                    # No directory hierarchy and the create failed telling us
                    # the container or volume directory does not exist. This
                    # could be a FUSE issue or some race condition, so let's
                    # sleep a bit and retry.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    continue
                if i != 0:
                    # Got ENOENT after previously making the path. This could
                    # also be a FUSE issue or some race condition, nap and
                    # retry.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs" % gerr)
                    continue
                # It looks like the path to the object does not already exist
                self._create_dir_object(self._obj_path)
                continue
            else:
                break
        else:
            # We failed after 1,000 attempts to create the temporary file.
            raise DiskFileError('DiskFile.mkstemp(): failed to successfully'
                                ' create a temporary file without running'
                                ' into a name conflict after 1,000 attempts'
                                ' for: %s' % (data_file,))

        self.tmppath = tmppath

        try:
            # Ensure it is properly owned before we make it available.
            do_fchown(fd, self.uid, self.gid)
            if _preallocate and size:
                # For XFS, fallocate() turns off speculative pre-allocation
                # until a write is issued either to the last block of the file
                # before the EOF or beyond the EOF. This means that we are
                # less likely to fragment free space with pre-allocated
                # extents that get truncated back to the known file size.
                # However, this call also turns holes into allocated but
                # unwritten extents, so that allocation occurs before the
                # write, not during XFS writeback. This effectively defeats
                # any allocation optimizations the filesystem can make at
                # writeback time.
                fallocate(fd, size)
            yield fd
        finally:
            try:
                do_close(fd)
            except OSError:
                pass
            if self.tmppath:
                tmppath, self.tmppath = self.tmppath, None
                do_unlink(tmppath)

     return error_response
 new_delete_at = int(request.headers.get('X-Delete-At') or 0)
 if new_delete_at and new_delete_at < time.time():
     return HTTPBadRequest(body='X-Delete-At in past', request=request,
                           content_type='text/plain')
 file = DiskFile(self.devices, device, partition, account, container,
                 obj, self.logger, disk_chunk_size=self.disk_chunk_size)
 orig_timestamp = file.metadata.get('X-Timestamp')
 upload_expiration = time.time() + self.max_upload_time
 etag = md5()
 upload_size = 0
 last_sync = 0
 elasped_time = 0
 with file.mkstemp() as fd:
     try:
         fallocate(fd, int(request.headers.get('content-length', 0)))
     except OSError:
         return HTTPInsufficientStorage(drive=device, request=request)
     reader = request.environ['wsgi.input'].read
     for chunk in iter(lambda: reader(self.network_chunk_size), ''):
         start_time = time.time()
         upload_size += len(chunk)
         if time.time() > upload_expiration:
             self.logger.increment('PUT.timeouts')
             return HTTPRequestTimeout(request=request)
         etag.update(chunk)
         while chunk:
             written = os.write(fd, chunk)
             chunk = chunk[written:]
         # For large files sync every 512MB (by default) written
         if upload_size - last_sync >= self.bytes_per_sync:

     return error_response
 new_delete_at = int(request.headers.get("X-Delete-At") or 0)
 if new_delete_at and new_delete_at < time.time():
     return HTTPBadRequest(body="X-Delete-At in past", request=request, content_type="text/plain")
 file = DiskFile(
     self.devices, device, partition, account, container, obj, self.logger, disk_chunk_size=self.disk_chunk_size
 )
 orig_timestamp = file.metadata.get("X-Timestamp")
 upload_expiration = time.time() + self.max_upload_time
 etag = md5()
 upload_size = 0
 last_sync = 0
 with file.mkstemp() as (fd, tmppath):
     if "content-length" in request.headers:
         try:
             fallocate(fd, int(request.headers["content-length"]))
         except OSError:
             return HTTPInsufficientStorage(drive=device, request=request)
     reader = request.environ["wsgi.input"].read
     for chunk in iter(lambda: reader(self.network_chunk_size), ""):
         upload_size += len(chunk)
         if time.time() > upload_expiration:
             self.logger.increment("PUT.timeouts")
             return HTTPRequestTimeout(request=request)
         etag.update(chunk)
         while chunk:
             written = os.write(fd, chunk)
             chunk = chunk[written:]
         # For large files sync every 512MB (by default) written
         if upload_size - last_sync >= self.bytes_per_sync:
             tpool.execute(fsync, fd)

#.........这里部分代码省略.........
        :raises DiskFileNoSpace: if a size is specified and allocation fails
        :raises AlreadyExistsAsFile: if path or part of a path is not a \
                                     directory
        """
        # Create /account/container directory structure on mount point root
        try:
            os.makedirs(self._container_path)
        except OSError as err:
            if err.errno != errno.EEXIST:
                raise

        data_file = os.path.join(self._put_datadir, self._obj)

        # Assume the full directory path exists to the file already, and
        # construct the proper name for the temporary file.
        attempts = 1
        while True:
            # To know more about why following temp file naming convention is
            # used, please read this GlusterFS doc:
            # https://github.com/gluster/glusterfs/blob/master/doc/features/dht.md#rename-optimizations  # noqa
            tmpfile = '.' + self._obj + '.' + uuid4().hex
            tmppath = os.path.join(self._put_datadir, tmpfile)
            try:
                fd = do_open(tmppath,
                             os.O_WRONLY | os.O_CREAT | os.O_EXCL | O_CLOEXEC)
            except SwiftOnFileSystemOSError as gerr:
                if gerr.errno in (errno.ENOSPC, errno.EDQUOT):
                    # Raise DiskFileNoSpace to be handled by upper layers when
                    # there is no space on disk OR when quota is exceeded
                    raise DiskFileNoSpace()
                if gerr.errno == errno.ENOTDIR:
                    raise AlreadyExistsAsFile('do_open(): failed on %s,'
                                              '  path or part of a'
                                              ' path is not a directory'
                                              % (tmppath))

                if gerr.errno not in (errno.ENOENT, errno.EEXIST, errno.EIO):
                    # FIXME: Other cases we should handle?
                    raise
                if attempts >= MAX_OPEN_ATTEMPTS:
                    # We failed after N attempts to create the temporary
                    # file.
                    raise DiskFileError('DiskFile.mkstemp(): failed to'
                                        ' successfully create a temporary file'
                                        ' without running into a name conflict'
                                        ' after %d of %d attempts for: %s' % (
                                            attempts, MAX_OPEN_ATTEMPTS,
                                            data_file))
                if gerr.errno == errno.EEXIST:
                    # Retry with a different random number.
                    attempts += 1
                elif gerr.errno == errno.EIO:
                    # FIXME: Possible FUSE issue or race condition, let's
                    # sleep on it and retry the operation.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    attempts += 1
                elif not self._obj_path:
                    # No directory hierarchy and the create failed telling us
                    # the container or volume directory does not exist. This
                    # could be a FUSE issue or some race condition, so let's
                    # sleep a bit and retry.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs", gerr)
                    attempts += 1
                elif attempts > 1:
                    # Got ENOENT after previously making the path. This could
                    # also be a FUSE issue or some race condition, nap and
                    # retry.
                    _random_sleep()
                    logging.warn("DiskFile.mkstemp(): %s ... retrying in"
                                 " 0.1 secs" % gerr)
                    attempts += 1
                else:
                    # It looks like the path to the object does not already
                    # exist; don't count this as an attempt, though, since
                    # we perform the open() system call optimistically.
                    self._create_dir_object(self._obj_path)
            else:
                break
        dw = None
        try:
            if size is not None and size > 0:
                try:
                    fallocate(fd, size)
                except OSError as err:
                    if err.errno in (errno.ENOSPC, errno.EDQUOT):
                        raise DiskFileNoSpace()
                    raise
            # Ensure it is properly owned before we make it available.
            do_fchown(fd, self._uid, self._gid)
            dw = DiskFileWriter(fd, tmppath, self)
            yield dw
        finally:
            if dw:
                dw.close()
                if dw._tmppath:
                    do_unlink(dw._tmppath)