Python filelike.StringIO类代码示例

 def __init__(self, client, name, max_buffer=MAX_BUFFER):
     self.client = client
     r = self.client.get_file(name)
     self.bytes = int(r.getheader('Content-Length'))
     if r > max_buffer:
         temp = tempfile.TemporaryFile()
     else:
         temp = StringIO()
     shutil.copyfileobj(r, temp)
     temp.seek(0)
     super(SpooledReader, self).__init__(temp, name)

 def setcontents(self, path, data, chunk_size=1024*64):
     if not isinstance(data, six.binary_type):        
         return super(MemoryFS, self).setcontents(path, data, chunk_size)        
     if not self.exists(path):      
         self.open(path, 'wb').close()                    
         
     dir_entry = self._get_dir_entry(path)
     if not dir_entry.isfile():
         raise ResourceInvalidError('Not a directory %(path)s', path)
     new_mem_file = StringIO()        
     new_mem_file.write(data)
     dir_entry.mem_file = new_mem_file                        

    def setcontents(self, path, data=b"", encoding=None, errors=None, chunk_size=1024 * 64):
        if isinstance(data, six.binary_type):
            if not self.exists(path):
                self.open(path, "wb").close()
            dir_entry = self._get_dir_entry(path)
            if not dir_entry.isfile():
                raise ResourceInvalidError("Not a directory %(path)s", path)
            new_mem_file = StringIO()
            new_mem_file.write(data)
            dir_entry.mem_file = new_mem_file
            return len(data)

        return super(MemoryFS, self).setcontents(
            path, data=data, encoding=encoding, errors=errors, chunk_size=chunk_size
        )

 def upload(self, path, f):
     import pdb; pdb.set_trace()
     if isinstance(f, basestring):
         # upload given string as file's contents.
         f = StringIO(f)
     l = None
     try:
         l = len(f)
     except:
         try:
             l = os.fstat(f.fileno()).st_size
         except:
             try:
                 f.seek(0, 2)
                 l = f.tell()
                 f.seek(0)
             except:
                 raise Exception('Could not determine length of file!')
     dirname, basename = pathsplit(path)
     try:
         info = self.info(path)
     except:
         try:
             info = self.info(dirname)
         except:
             raise Exception('Cannot upload to non-existent directory!')
     url = '%s/%s/%s' % (ULURL, self.auth_token, info['inode'])
     host = urlparse.urlparse(url).hostname
     conn = httplib.HTTPConnection(host, 443)
     boundary = mimetools.choose_boundary()
     fields = {
         'boundary': boundary,
         'mime': mimetypes.guess_type(basename)[0] or 'application/octet-stream',
         'name': basename,
     }
     head = MULTIPART_HEAD % fields
     tail = MULTIPART_TAIL % fields
     l += len(head) + len(tail)
     headers = {
         'Content-Length': l,
         'Content-Type': 'multipart/form-data; boundary=%s' % boundary,
     }
     conn.request('POST', url, '', headers)
     # now stream the file to box.net.
     conn.send(head)
     while True:
         data = f.read(4096)
         if not data:
             break
         conn.send(data)
     conn.send(tail)
     r = conn.getresponse()
     if r.status != 200:
         raise Exception('Error uploading data!')

 def __setstate__(self, state):
     self.__dict__.update(state)
     if self.type == 'file':
         self.lock = threading.RLock()
     else:
         self.lock = None
     if self.mem_file is not None:
         data = self.mem_file
         self.mem_file = StringIO()
         self.mem_file.write(data)

    def open(self, path, mode='r', buffering=-1, encoding=None, errors=None, newline=None, line_buffering=False, **kwargs):
        # TODO: chunked transport of large files
        epath = self.encode_path(path)
        if "w" in mode:
            self.proxy.set_contents(epath, xmlrpclib.Binary(b("")))
        if "r" in mode or "a" in mode or "+" in mode:
            try:
                data = self.proxy.get_contents(epath, "rb").data
            except IOError:
                if "w" not in mode and "a" not in mode:
                    raise ResourceNotFoundError(path)
                if not self.isdir(dirname(path)):
                    raise ParentDirectoryMissingError(path)
                self.proxy.set_contents(path, xmlrpclib.Binary(b("")))
        else:
            data = b("")
        f = StringIO(data)
        if "a" not in mode:
            f.seek(0, 0)
        else:
            f.seek(0, 2)
        oldflush = f.flush
        oldclose = f.close
        oldtruncate = f.truncate

        def newflush():
            self._lock.acquire()
            try:
                oldflush()
                self.proxy.set_contents(epath, xmlrpclib.Binary(f.getvalue()))
            finally:
                self._lock.release()

        def newclose():
            self._lock.acquire()
            try:
                f.flush()
                oldclose()
            finally:
                self._lock.release()

        def newtruncate(size=None):
            self._lock.acquire()
            try:
                oldtruncate(size)
                f.flush()
            finally:
                self._lock.release()

        f.flush = newflush
        f.close = newclose
        f.truncate = newtruncate
        return f

    def __init__(self, fs, path, mode, rfile=None, write_on_flush=True):
        """RemoteFileBuffer constructor.

        The owning filesystem, path and mode must be provided.  If the
        optional argument 'rfile' is provided, it must be a read()-able
        object or a string containing the initial file contents.
        """
        wrapped_file = SpooledTemporaryFile(max_size=self.max_size_in_memory)
        self.fs = fs
        self.path = path
        self.write_on_flush = write_on_flush
        self._changed = False
        self._readlen = 0  # How many bytes already loaded from rfile
        self._rfile = None  # Reference to remote file object
        self._eof = False  # Reached end of rfile?
        if getattr(fs, "_lock", None) is not None:
            self._lock = fs._lock.__class__()
        else:
            self._lock = threading.RLock()

        if "r" in mode or "+" in mode or "a" in mode:
            if rfile is None:
                # File was just created, force to write anything
                self._changed = True
                self._eof = True

            if not hasattr(rfile, "read"):
                #rfile = StringIO(unicode(rfile))
                rfile = StringIO(rfile)

            self._rfile = rfile
        else:
            # Do not use remote file object
            self._eof = True
            self._rfile = None
            self._changed = True
            if rfile is not None and hasattr(rfile,"close"):
                rfile.close()
        super(RemoteFileBuffer,self).__init__(wrapped_file,mode)
        # FIXME: What if mode with position on eof?
        if "a" in mode:
            # Not good enough...
            self.seek(0, SEEK_END)

 def __setstate__(self, state):
     state["bucket"] = RiakBucket(state.pop("bucket"), state.pop("host"), state.pop("port"), state.pop("transport"))
     self.__dict__.update(state)
     if self.type == "file":
         self.lock = threading.RLock()
     else:
         self.lock = None
     if self._mem_file is not None:
         data = self._mem_file
         self._mem_file = StringIO()
         self._mem_file.write(data)

 def open(self, path, mode="r"):
     # TODO: chunked transport of large files
     path = self.encode_path(path)
     if "w" in mode:
         self.proxy.set_contents(path,xmlrpclib.Binary(""))
     if "r" in mode or "a" in mode or "+" in mode:
         try:
             data = self.proxy.get_contents(path).data
         except IOError:
             if "w" not in mode and "a" not in mode:
                 raise ResourceNotFoundError(path)
             if not self.isdir(dirname(path)):
                 raise ParentDirectoryMissingError(path)
             self.proxy.set_contents(path,xmlrpclib.Binary(""))
     else:
         data = ""
     f = StringIO(data)
     if "a" not in mode:
         f.seek(0,0)
     else:
         f.seek(0,2)
     oldflush = f.flush
     oldclose = f.close
     oldtruncate = f.truncate
     def newflush():
         oldflush()
         self.proxy.set_contents(path,xmlrpclib.Binary(f.getvalue()))
     def newclose():
         f.flush()
         oldclose()
     def newtruncate(size=None):
         oldtruncate(size)
         f.flush()
     f.flush = newflush
     f.close = newclose
     f.truncate = newtruncate
     return f

    def __init__(self, type, name, contents=None):

        assert type in ("dir", "file"), "Type must be dir or file!"

        self.type = type
        self.name = name

        if contents is None and type == "dir":
            contents = {}

        self.open_files = []
        self.contents = contents
        self.mem_file = None
        self.created_time = datetime.datetime.now()
        self.modified_time = self.created_time
        self.accessed_time = self.created_time

        self.xattrs = {}

        self.lock = None
        if self.type == 'file':
            self.mem_file = StringIO()
            self.lock = threading.RLock()

class DirEntry(object):

    def sync(f):
        def deco(self, *args, **kwargs):
            if self.lock is not None:
                try:
                    self.lock.acquire()
                    return f(self, *args, **kwargs)
                finally:
                    self.lock.release()
            else:
                return f(self, *args, **kwargs)
        return deco

    def __init__(self, type, name, contents=None):

        assert type in ("dir", "file"), "Type must be dir or file!"

        self.type = type
        self.name = name

        if contents is None and type == "dir":
            contents = {}

        self.open_files = []
        self.contents = contents
        self.mem_file = None
        self.created_time = datetime.datetime.now()
        self.modified_time = self.created_time
        self.accessed_time = self.created_time

        self.xattrs = {}

        self.lock = None
        if self.type == 'file':
            self.mem_file = StringIO()
            self.lock = threading.RLock()

    def get_value(self):
        self.lock.acquire()
        try:
            return self.mem_file.getvalue()
        finally:
            self.lock.release()
    data = property(get_value)

    def desc_contents(self):
        if self.isfile():
            return "<file %s>" % self.name
        elif self.isdir():
            return "<dir %s>" % "".join("%s: %s" % (k, v.desc_contents()) for k, v in self.contents.iteritems())

    def isdir(self):
        return self.type == "dir"

    def isfile(self):
        return self.type == "file"

    def __str__(self):
        return "%s: %s" % (self.name, self.desc_contents())

    @sync
    def __getstate__(self):
        state = self.__dict__.copy()
        state.pop('lock')
        if self.mem_file is not None:
            state['mem_file'] = self.data
        return state

    def __setstate__(self, state):
        self.__dict__.update(state)
        if self.type == 'file':
            self.lock = threading.RLock()
        else:
            self.lock = None
        if self.mem_file is not None:
            data = self.mem_file
            self.mem_file = StringIO()
            self.mem_file.write(data)

class RiakFSObject(DirEntry):
    """
    Represents a filesystem "node", either a directory of file.

    A directory node may have sub-nodes in a contents dictionary.

    Has more responsibility than the `DirEntry` class, for example the
    `remove` and `_make_dir_entry` methods. Also has a `path` attribute
    which, in the case of a file, is the key of the object in the Riak
    store. TODO: look into moving the responsibilty back to the FS object,
    make this class dumber.
    """

    @classmethod
    def from_dict(cls, bucket, data):
        def obj_from_dict(d):
            type = d.pop("type")
            name = d.pop("name")
            prefix = d.pop("prefix", None)
            contents = d.pop("contents", {})
            obj = cls(bucket, type, name, prefix)
            obj.xattrs = d["xattrs"]
            obj.timestamps = d["timestamps"]
            for k, v in contents.items():
                obj.contents[k] = obj_from_dict(v)
            return obj

        return obj_from_dict(data)

    def to_dict(self):
        ignore = set(["bucket", "contents", "lock", "open_files"])

        def serialize(obj):
            d = {}
            for k, v in obj.__dict__.iteritems():
                if k in ignore or k.startswith("_"):
                    continue
                if k == "xattrs" and not v:
                    continue
                d[k] = v
            if obj.contents:
                d["contents"] = dict((k, serialize(v)) for k, v in obj.contents.items())
            return d

        return serialize(self)

    # Datetime instances don't json serialize, so we maintain them as lists
    # under the hood and "rehydrate" on demand
    def _get_ct(self):
        return datetime.fromtimestamp(time.mktime(self.timestamps["ctime"]))

    def _get_mt(self):
        return datetime.fromtimestamp(time.mktime(self.timestamps["mtime"]))

    def _get_at(self):
        return datetime.fromtimestamp(time.mktime(self.timestamps["atime"]))

    def _set_ct(self, val):
        self.timestamps["ctime"] = list(val.timetuple())

    def _set_mt(self, val):
        self.timestamps["mtime"] = list(val.timetuple())

    def _set_at(self, val):
        self.timestamps["atime"] = list(val.timetuple())

    created_time = property(_get_ct, _set_ct)
    modified_time = property(_get_mt, _set_mt)
    accessed_time = property(_get_at, _set_at)

    def _get_file(self):
        if self.type == "file" and self._mem_file is None:
            bytes = self.bucket.get_binary(self.path).get_data()
            self._mem_file = StringIO(bytes)
        return self._mem_file

    def _set_file(self, stream):
        self._mem_file = stream

    mem_file = property(_get_file, _set_file)

    def __init__(self, bucket, type, name, prefix=None, contents=None):
        assert type in ("dir", "file"), "Type must be dir or file!"
        self.bucket = bucket
        self.type = type
        self.name = name.rstrip("/")
        if prefix:
            prefix = prefix.strip("/") + "/"
        else:
            prefix = ""
        self.prefix = prefix
        self.path = prefix + name
        if type == "dir":
            self.path += "/"
            if contents is None:
                contents = {}
        self.open_files = []
        self.contents = contents

        now = list(datetime.now().timetuple())
#.........这里部分代码省略.........

 def _get_file(self):
     if self.type == "file" and self._mem_file is None:
         bytes = self.bucket.get_binary(self.path).get_data()
         self._mem_file = StringIO(bytes)
     return self._mem_file