Python chunkobject.ChunkObject类代码示例

    def __init__(self, cffs, container, obj, mode):
        self.cffs = cffs
        self.container = container
        self.name = obj
        self.mode = mode
        self.closed = False
        self.total_size = 0
        self.headers = dict()

        self.obj = None

        # this is only used by `seek`, so we delay the HEAD request until is required
        self.size = None

        if not all([container, obj]):
            self.closed = True
            raise IOSError(EPERM, 'Container and object required')

        logging.debug("ObjectStorageFD object: %r (mode: %r)" % (obj, mode))

        if 'r' in self.mode:
            logging.debug("read fd %r" % self.name)
        else: # write
            logging.debug("write fd %r" % self.name)
            self.obj = ChunkObject(self.conn, self.container, self.name, content_type=mimetypes.guess_type(self.name)[0])

    def write(self, data):
        """Write data to the object."""
        if 'r' in self.mode:
            raise IOSError(EPERM, "File is opened for read")

        # large file support
        if self.split_size:
            # data can be of any size, so we need to split it in split_size chunks
            offs = 0
            while offs < len(data):
                if self.part_size + len(data) - offs > self.split_size:
                    current_size = self.split_size-self.part_size
                    logging.debug("data is to large (%r), using %s" % (len(data), current_size))
                else:
                    current_size = len(data)-offs
                self.part_size += current_size
                if not self.obj:
                    self.obj = ChunkObject(self.conn, self.container, self.part_name, content_type=self.content_type)
                self.obj.send_chunk(data[offs:offs+current_size])
                offs += current_size
                if self.part_size == self.split_size:
                    logging.debug("current size is %r, split_file is %r" % (self.part_size, self.split_size))
                    self.obj.finish_chunk()
                    # this obj is not valid anymore, will create a new one if a new part is required
                    self.obj = None
                    # make it the first part
                    if self.part == 0:
                        self._start_copy_task()
                    self.part_size = 0
                    self.part += 1
        else:
            self.obj.send_chunk(data)

    def __init__(self, connection, container, obj, mode):
        self.conn = connection
        self.container = container
        self.name = obj
        self.mode = mode
        self.closed = False
        self.total_size = 0
        self.part_size = 0
        self.part = 0
        self.part_collision = 0
        self.headers = dict()
        self.content_type = mimetypes.guess_type(self.name)[0]
        self.pending_copy_task = None

        self.obj = None

        # this is only used by `seek`, so we delay the HEAD request until is required
        self.size = None

        if not all([container, obj]):
            self.closed = True
            raise IOSError(EPERM, 'Container and object required')

        logging.debug("ObjectStorageFD object: %r (mode: %r)" % (obj, mode))

        if 'r' in self.mode:
            logging.debug("read fd %r" % self.name)
        else: # write
            logging.debug("write fd %r" % self.name)

            # check for collisions in case this is a multi-part file
            if self.split_size:
                self._find_collisions()

            self.obj = ChunkObject(self.conn, self.container, self.name, content_type=self.content_type)

class CloudFilesFD(object):
    '''Acts like a file() object, but attached to a cloud files object'''

    def __init__(self, cffs, container, obj, mode):
        self.cffs = cffs
        self.container = container
        self.name = obj
        self.mode = mode
        self.closed = False
        self.total_size = 0
        self.stream = None

        if not all([container, obj]):
            self.closed = True
            raise IOSError(EPERM, 'Container and object requred')

        self.container = self.cffs._get_container(self.container)

        if 'r' in self.mode:
            self.obj = self.container.get_object(self.name)
            logging.debug("read fd obj.name=%r obj.size=%r" % (self.obj.name, self.obj.size))
        else: #write
            self.obj = ChunkObject(self.container, obj)
            self.obj.content_type = mimetypes.guess_type(obj)[0]
            self.obj.prepare_chunk()

    def write(self, data):
        '''Write data to the object'''
        if 'r' in self.mode:
            raise IOSError(EPERM, "Can't write to stream opened for read")
        self.obj.send_chunk(data)

    def close(self):
        '''Close the object and finish the data transfer'''
        if 'r' in self.mode:
            return
        self.obj.finish_chunk()

    def read(self, size=65536):
        '''Read data from the object.

        We can use just one request because 'seek' is not supported.
        
        NB: It uses the size passed into the first call for all subsequent calls'''
        if not self.stream:
            self.stream = self.obj.stream(size)

        logging.debug("read size=%r, total_size=%r, obj.size=%r" % (size, self.total_size, self.obj.size))
        try:
            buff = self.stream.next()
            self.total_size += len(buff)
        except StopIteration:
            return ""
        else:
            return buff

    def seek(self, *kargs, **kwargs):
        '''Seek in the object: FIXME doesn't work and raises an error'''
        logging.debug("seek args=%s, kargs=%s" % (str(kargs), str(kwargs)))
        raise IOSError(EPERM, "Seek not implemented")

class CloudFilesFD(object):
    """Acts like a file() object, but attached to a cloud files object"""

    def __init__(self, cffs, container, obj, mode):
        self.cffs = cffs
        self.container = container
        self.name = obj
        self.mode = mode
        self.closed = False
        self.total_size = 0

        if not all([container, obj]):
            self.closed = True
            raise IOSError(EPERM, "Container and object requred")

        self.container = self.cffs._get_container(self.container)

        if "r" in self.mode:
            self.obj = self.container.get_object(self.name)
            logging.debug("read fd obj.name=%r obj.size=%r" % (self.obj.name, self.obj.size))
        else:  # write
            self.obj = ChunkObject(self.container, obj)
            self.obj.content_type = mimetypes.guess_type(obj)[0]
            self.obj.prepare_chunk()

    def write(self, data):
        """Write data to the object"""
        if "r" in self.mode:
            raise IOSError(EPERM, "Can't write to stream opened for read")
        self.obj.send_chunk(data)

    def close(self):
        """Close the object and finish the data transfer"""
        if "r" in self.mode:
            return
        self.obj.finish_chunk()

    def read(self, size=65536):
        """Read data from the object"""
        logging.debug("read size=%r, total_size=%r, obj.size=%r" % (size, self.total_size, self.obj.size))
        readsize = size
        if (self.total_size + size) > self.obj.size:
            readsize = self.obj.size - self.total_size
        if self.total_size >= self.obj.size:
            return ""
        else:
            offset = self.total_size
            self.total_size += size
            return self.obj.read(size=readsize, offset=offset)

    def seek(self, *kargs, **kwargs):
        """Seek in the object: FIXME doesn't work and raises an error"""
        logging.debug("seek args=%s, kargs=%s" % (str(kargs), str(kwargs)))
        raise IOSError(EPERM, "Seek not implemented")

    def __init__(self, cffs, container, obj, mode):
        self.cffs = cffs
        self.container = container
        self.name = obj
        self.mode = mode
        self.closed = False
        self.total_size = 0

        if not all([container, obj]):
            self.closed = True
            raise IOSError(EPERM, "Container and object requred")

        self.container = self.cffs._get_container(self.container)

        if "r" in self.mode:
            self.obj = self.container.get_object(self.name)
            logging.debug("read fd obj.name=%r obj.size=%r" % (self.obj.name, self.obj.size))
        else:  # write
            self.obj = ChunkObject(self.container, obj)
            self.obj.content_type = mimetypes.guess_type(obj)[0]
            self.obj.prepare_chunk()

class ObjectStorageFD(object):
    """File alike object attached to the Object Storage."""
    def __init__(self, cffs, container, obj, mode):
        self.cffs = cffs
        self.container = container
        self.name = obj
        self.mode = mode
        self.closed = False
        self.total_size = 0
        self.headers = dict()

        self.obj = None

        # this is only used by `seek`, so we delay the HEAD request until is required
        self.size = None

        if not all([container, obj]):
            self.closed = True
            raise IOSError(EPERM, 'Container and object required')

        logging.debug("ObjectStorageFD object: %r (mode: %r)" % (obj, mode))

        if 'r' in self.mode:
            logging.debug("read fd %r" % self.name)
        else: # write
            logging.debug("write fd %r" % self.name)
            self.obj = ChunkObject(self.conn, self.container, self.name, content_type=mimetypes.guess_type(self.name)[0])

    @property
    def conn(self):
        """Connection to the storage."""
        return self.cffs.conn

    def write(self, data):
        """Write data to the object."""
        if 'r' in self.mode:
            raise IOSError(EPERM, "File is opened for read")
        self.obj.send_chunk(data)

    def close(self):
        """Close the object and finish the data transfer."""
        if 'r' in self.mode:
            return
        self.obj.finish_chunk()

    def read(self, size=65536):
        """
        Read data from the object.

        We can use just one request because 'seek' is not fully supported.

        NB: It uses the size passed into the first call for all subsequent calls.
        """
        if self.obj is None:
            if self.total_size > 0:
                self.conn.range_from = self.total_size
                # we need to open a new connection to inject the `Range` header
                if self.conn.http_conn:
                    self.conn.http_conn[1].close()
                    self.conn.http_conn = None
            _, self.obj = self.conn.get_object(self.container, self.name, resp_chunk_size=size)

        logging.debug("read size=%r, total_size=%r (range_from: %s)" % (size,
                self.total_size, self.conn.range_from))

        try:
            buff = self.obj.next()
            self.total_size += len(buff)
        except StopIteration:
            return ""
        else:
            return buff

    def seek(self, offset, whence=None):
        """
        Seek in the object.

        It's supported only for read operations because of object storage limitations.
        """
        logging.debug("seek offset=%s, whence=%s" % (str(offset), str(whence)))

        if 'r' in self.mode:

            if self.size is None:
                meta = self.conn.head_object(self.container, self.name)
                try:
                    self.size = int(meta["content-length"])
                except ValueError:
                    raise IOSError(EPERM, "Invalid file size")

            if not whence:
                offs = offset
            elif whence == 1:
                offs = self.total_size + offset
            elif whence == 2:
                offs = self.size - offset
            else:
                raise IOSError(EPERM, "Invalid file offset")

            if offs < 0 or offs > self.size:
#.........这里部分代码省略.........

class ObjectStorageFD(object):
    """File alike object attached to the Object Storage."""

    split_size = 0

    def __init__(self, cffs, container, obj, mode):
        self.cffs = cffs
        self.container = container
        self.name = obj
        self.mode = mode
        self.closed = False
        self.total_size = 0
        self.part_size = 0
        self.part = 0
        self.headers = dict()
        self.content_type = mimetypes.guess_type(self.name)[0]
        self.pending_copy_task = None

        self.obj = None

        # this is only used by `seek`, so we delay the HEAD request until is required
        self.size = None

        if not all([container, obj]):
            self.closed = True
            raise IOSError(EPERM, 'Container and object required')

        logging.debug("ObjectStorageFD object: %r (mode: %r)" % (obj, mode))

        if 'r' in self.mode:
            logging.debug("read fd %r" % self.name)
        else: # write
            logging.debug("write fd %r" % self.name)
            self.obj = ChunkObject(self.conn, self.container, self.name, content_type=self.content_type)

    @property
    def part_base_name(self):
        return u"%s.part" % self.name

    @property
    def part_name(self):
        return u"%s/%.6d" % (self.part_base_name, self.part)

    @property
    def conn(self):
        """Connection to the storage."""
        return self.cffs.conn

    def _start_copy_task(self):
        """
        Copy the first part of a multi-part file to its final location and create
        the manifest file.

        This happens in the background, pending_copy_task must be cleaned up at
        the end.
        """
        def copy_task(conn, container, name, part_name, part_base_name):
            # open a new connection
            conn = ProxyConnection(None, preauthurl=conn.url, preauthtoken=conn.token)
            headers = { 'x-copy-from': "/%s/%s" % (container, name) }
            logging.debug("copying first part %r/%r, %r" % (container, part_name, headers))
            try:
                conn.put_object(container, part_name, headers=headers, contents=None)
            except ClientException as ex:
                logging.error("Failed to copy %s: %s" % (name, ex.http_reason))
                sys.exit(1)
            # setup the manifest
            headers = { 'x-object-manifest': "%s/%s" % (container, part_base_name) }
            logging.debug("creating manifest %r/%r, %r" % (container, name, headers))
            try:
                conn.put_object(container, name, headers=headers, contents=None)
            except ClientException as ex:
                logging.error("Failed to store the manifest %s: %s" % (name, ex.http_reason))
                sys.exit(1)
            logging.debug("copy task done")
        self.pending_copy_task = multiprocessing.Process(target=copy_task,
                                                         args=(self.conn,
                                                               self.container,
                                                               self.name,
                                                               self.part_name,
                                                               self.part_base_name,
                                                               ),
                                                         )
        self.pending_copy_task.start()

    @translate_objectstorage_error
    def write(self, data):
        """Write data to the object."""
        if 'r' in self.mode:
            raise IOSError(EPERM, "File is opened for read")

        # large file support
        if self.split_size:
            # data can be of any size, so we need to split it in split_size chunks
            offs = 0
            while offs < len(data):
                if self.part_size + len(data) - offs > self.split_size:
                    current_size = self.split_size-self.part_size
                    logging.debug("data is to large (%r), using %s" % (len(data), current_size))
                else:
#.........这里部分代码省略.........

class CloudFilesFD(object):
    '''Acts like a file() object, but attached to a cloud files object'''

    def __init__(self, cffs, container, obj, mode):
        self.cffs = cffs
        self.container = container
        self.name = obj
        self.mode = mode
        self.closed = False
        self.total_size = 0
        self.stream = None
        self.headers = dict()

        if not all([container, obj]):
            self.closed = True
            raise IOSError(EPERM, 'Container and object required')

        self.container = self.cffs._get_container(self.container)

        if 'r' in self.mode:
            self.obj = self.container.get_object(self.name)
            logging.debug("read fd obj.name=%r obj.size=%r" % (self.obj.name, self.obj.size))
        else: #write
            self.obj = ChunkObject(self.container, obj)
            self.obj.content_type = mimetypes.guess_type(obj)[0]
            self.obj.prepare_chunk()

    def write(self, data):
        '''Write data to the object'''
        if 'r' in self.mode:
            raise IOSError(EPERM, "Can't write to stream opened for read")
        self.obj.send_chunk(data)

    def close(self):
        '''Close the object and finish the data transfer'''
        if 'r' in self.mode:
            return
        self.obj.finish_chunk()

    def read(self, size=65536):
        '''Read data from the object.

        We can use just one request because 'seek' is not fully supported.

        NB: It uses the size passed into the first call for all subsequent calls'''
        if not self.stream:
            self.stream = self.obj.stream(size, hdrs=self.headers)

        logging.debug("read size=%r, total_size=%r, obj.size=%r" % (size, self.total_size, self.obj.size))
        try:
            buff = self.stream.next()
            self.total_size += len(buff)
        except StopIteration:
            return ""
        else:
            return buff

    def seek(self, offset, whence=None):
        '''Seek in the object.

        It's supported only for read operations because of the object storage limitations.'''
        logging.debug("seek offset=%s, whence=%s" % (str(offset), str(whence)))
        if 'r' in self.mode:
            if not whence:
                offs = offset
            elif whence == 1:
                offs = self.total_size + offset
            elif whence == 2:
                offs = self.obj.size - offset
            else:
                raise IOSError(EPERM, "Invalid file offset")

            if offs < 0 or offs > self.obj.size:
                raise IOSError(EPERM, "Invalid file offset")

            # we need to start over after a seek call
            if self.stream:
                self.stream = None
                self.obj = self.container.get_object(self.name)
            self.headers['Range'] = "bytes=%s-" % offs
            self.total_size = offs
        else:
            raise IOSError(EPERM, "Seek not available for write operations")