Python SpooledTemporaryFile.flush方法代码示例

# 需要导入模块: from tempfile import SpooledTemporaryFile [as 别名]
# 或者: from tempfile.SpooledTemporaryFile import flush [as 别名]
    def flush(self):
        """Flush the data to the server."""

        SpooledTemporaryFile.flush(self)
        self.commit()

# 需要导入模块: from tempfile import SpooledTemporaryFile [as 别名]
# 或者: from tempfile.SpooledTemporaryFile import flush [as 别名]
    def fetch_media(self, url, partial_fetch=False):
        """Retrieves a given media object from a remote (HTTP) location
        and returns the content-type and a file-like object containing
        the media content.

        The file-like object is a temporary file that - depending on the
        size - lives in memory or on disk. Once the file is closed, the
        contents are removed from storage.

        :param url: the URL of the media asset.
        :type url: str.
        :param partial_fetch: determines if the the complete file should
            be fetched, or if only the first 2 MB should be retrieved.
            This feature is used to prevent complete retrieval of large
            a/v material.
        :type partial_fetch: bool.
        :returns: a tuple with the ``content-type``, ``content-lenght``
            and a file-like object containing the media content. The
            value of ``content-length`` will be ``None`` in case
            a partial fetch is requested and ``content-length`` is not
            returned by the remote server.
        """

        http_resp = self.http_session.get(url, stream=True, timeout=(60, 120))
        http_resp.raise_for_status()

        if not os.path.exists(TEMP_DIR_PATH):
            log.debug('Creating temp directory %s' % TEMP_DIR_PATH)
            os.makedirs(TEMP_DIR_PATH)

        # Create a temporary file to store the media item, write the file
        # to disk if it is larger than 1 MB.
        media_file = SpooledTemporaryFile(max_size=1024*1024, prefix='ocd_m_',
                                          suffix='.tmp',
                                          dir=TEMP_DIR_PATH)

        # When a partial fetch is requested, request up to two MB
        partial_target_size = 1024*1024*2
        content_length = http_resp.headers.get('content-length')
        if content_length and int(content_length) < partial_target_size:
            partial_target_size = int(content_length)

        retrieved_bytes = 0
        for chunk in http_resp.iter_content(chunk_size=512*1024):
            if chunk:  # filter out keep-alive chunks
                media_file.write(chunk)
                retrieved_bytes += len(chunk)

            if partial_fetch and retrieved_bytes >= partial_target_size:
                break

        media_file.flush()
        log.debug('Fetched media item %s [%s/%s]' % (url, retrieved_bytes,
                                                     content_length))

        # If the server doens't provide a content-length and this isn't
        # a partial fetch, determine the size by looking at the retrieved
        # content
        if not content_length and not partial_fetch:
            media_file.seek(0, 2)
            content_length = media_file.tell()

        return (
            http_resp.headers.get('content-type'),
            content_length,
            media_file
        )

# 需要导入模块: from tempfile import SpooledTemporaryFile [as 别名]
# 或者: from tempfile.SpooledTemporaryFile import flush [as 别名]
class RecvWindow(object):
    """
    Receiver windows receive data on a socket to fill their
    buffer.

    A window instance has the following attributes:

        name        Name of the window. The name of all windows is shown by
                    '> win show' in command line interface. It's usually the
                    name of the plugin appended by the arguments passed for
                    execution (e.g. ps.plg -A -t curesec.com)
        status      indicates the state of the plugin (e.g. RUNNING or FINISHED)
        req_t       request tuple that triggered plugin execution. It's used to
                    verify incoming packets
        plg_id      id of the plugin
        rid         request id of the packet triggering execution
        wid         the id of the window. Derived from a class variable which
                    is increased after window creation. The window id is not
                    fixed. So every new client session, the windows might have
                    a new id.
        session     the actual client session
        is_used     flag, that indicates, whether the windows is used by user
                    (via '> win show <id>')

    """

    #: command to close window
    CMD_CLOSE = "close"

    #: window id, increased for every new window
    ID = 1

    def __init__(self, name, req_t, session):
        logger.debug("creating window %s", name)
        self.name = name
        self.status = RUNNING
        self.req_t = req_t
        self.sid = req_t[1]
        self.rid = req_t[2]
        self.plg_id = req_t[4]
        self.wid = RecvWindow.ID
        self.session = session
        self.is_used = False
        self.closed = False

        self._buffer = SpooledTemporaryFile(max_size="8096", mode="wr+b")
        RecvWindow.ID += 1

    def close(self):
        self.closed = True
        print("closing window %d" % self.wid)

    def set_state(self, input_code):
        state = None
        try:
            state = STATE[input_code + 1]
        except (IndexError, TypeError) as e:
            logger.warning('Could not get window state for code %d: %s', input_code, e)
            logger.exception(e)

        if not state:
            state = "Unknown"

        if self.is_used and state in (FINISHED, ERROR, KILLED):
            self.is_used = False

        code = "(%s)" % str(input_code)
        self.status = "{state} {code}".format(state=state, code=code)

        logger.debug("new window(id:%s) state: %s", self.wid, self.status)

    def _write_to_output(self, data):
        # if no more data is coming, set window state to
        # finished and break if we have a non-interactive
        # window (otherwise we need to wait for the user
        # to enter further input).
        #if not data:
        #    self.status = FINISHED
        # TODO: that seems a bit heuristic to me ...

        # ok, we have data, so verify 'em and print it. this is
        # done by unpack
        try:
            actual_output = self._unpack(data)
            if actual_output:
                sys.stdout.write(actual_output)
                sys.stdout.flush()
                self._buffer.write(actual_output)
                self._buffer.flush()

        except errors.ClientError as e:
            if e.request:
                logger.warning('Request: %s', conn.ccdlib.pkt2str(e.request))
            if e.response:
                logger.warning('Response: %s', conn.ccdlib.pkt2str(e.response))
            logger.debug("received invalid packet:'%s'", e)
            logger.exception(e)
        except Exception as e:
            logger.debug("received invalid packet:'%s'", e)
            logger.exception(e)
#.........这里部分代码省略.........

# 需要导入模块: from tempfile import SpooledTemporaryFile [as 别名]
# 或者: from tempfile.SpooledTemporaryFile import flush [as 别名]
class Buffer(FileWrapper):
    """Class implementing buffereing of input and output streams.
    
    This class uses a separate buffer file to hold the contents of the
    underlying file while they are being manipulated.  As data is read
    it is duplicated into the buffer, and data is written from the buffer
    back to the file on close.
    """
    
    def __init__(self,fileobj,mode=None,max_size_in_memory=1024*8):
        """Buffered file wrapper constructor."""
        self._buffer = SpooledTemporaryFile(max_size=max_size_in_memory)
        self._in_eof = False
        self._in_pos = 0
        super(Buffer,self).__init__(fileobj,mode)

    def _buffer_chunks(self):
        chunk = self._buffer.read(16*1024)
        if chunk == "":
            yield chunk
        else:
            while chunk != "":
                yield chunk
                chunk = self._buffer.read(16*1024)

    def _write_out_buffer(self):
        if self._check_mode("r"):
            self._read_rest()
            if "a" in self.mode:
                self._buffer.seek(self._in_pos)
                self._fileobj.seek(self._in_pos)
            else:
                self._fileobj.seek(0)
                self._buffer.seek(0)
        else:
            self._buffer.seek(0)
        for chunk in self._buffer_chunks():
            self._fileobj.write(chunk)
 
    def flush(self):
        # flush the buffer; we only write to the underlying file on close
        self._buffer.flush()

    def close(self):
        if self.closed:
            return
        if self._check_mode("w"):
            self._write_out_buffer()
        super(Buffer,self).close()
        self._buffer.close()

    def _read(self,sizehint=-1):
        #  First return any data available from the buffer.
        #  Since we don't flush the buffer after every write, certain OSes
        #  (guess which!) will happy read junk data from the end of it.
        #  Instead, we explicitly read only up to self._in_pos.
        if not self._in_eof:
            buffered_size = self._in_pos - self._buffer.tell()
            if sizehint >= 0:
                buffered_size = min(sizehint,buffered_size)
        else:
            buffered_size = sizehint
        data = self._buffer.read(buffered_size)
        if data != "":
            return data
        # Then look for more data in the underlying file
        if self._in_eof:
            return None
        data = self._fileobj.read(sizehint)
        self._in_pos += len(data)
        self._buffer.write(data)
        if sizehint < 0 or len(data) < sizehint:
            self._in_eof = True
            self._buffer.flush()
        return data

    def _write(self,data,flushing=False):
        self._buffer.write(data)
        if self._check_mode("r") and not self._in_eof:
            diff = self._buffer.tell() - self._in_pos
            if diff > 0:
                junk = self._fileobj.read(diff)
                self._in_pos += len(junk)
                if len(junk) < diff:
                    self._in_eof = True
                    self._buffer.flush()
    
    def _seek(self,offset,whence):
        # Ensure we've read enough to simply do the seek on the buffer
        if self._check_mode("r") and not self._in_eof:
            if whence == 0:
                if offset > self._in_pos:
                    self._read_rest()
            if whence == 1:
                if self._buffer.tell() + offset > self._in_pos:
                    self._read_rest()
            if whence == 2:
                self._read_rest()
        # Then just do it on the buffer...
        self._buffer.seek(offset,whence)
#.........这里部分代码省略.........

# 需要导入模块: from tempfile import SpooledTemporaryFile [as 别名]
# 或者: from tempfile.SpooledTemporaryFile import flush [as 别名]
class Buffer(FileWrapper):
    """Class implementing buffering of input and output streams.
    
    This class uses a separate buffer file to hold the contents of the
    underlying file while they are being manipulated.  As data is read
    it is duplicated into the buffer, and data is written from the buffer
    back to the file on close.
    """

    def __init__(self, fileobj, mode=None, max_size_in_memory=1024 * 8):
        """Buffered file wrapper constructor."""
        self._buffer = SpooledTemporaryFile(max_size=max_size_in_memory)
        self._in_eof = False
        self._in_pos = 0
        self._was_truncated = False
        super(Buffer, self).__init__(fileobj, mode)

    def _buffer_size(self):
        try:
            return len(self._buffer.file.getvalue())
        except AttributeError:
            return os.fstat(self._buffer.fileno()).st_size

    def _buffer_chunks(self):
        chunk = self._buffer.read(16 * 1024)
        if chunk == "":
            yield chunk
        else:
            while chunk != "":
                yield chunk
                chunk = self._buffer.read(16 * 1024)

    def _write_out_buffer(self):
        if self._check_mode("r"):
            self._read_rest()
            if "a" in self.mode:
                self._buffer.seek(self._in_pos)
                self._fileobj.seek(self._in_pos)
            else:
                self._fileobj.seek(0)
                self._buffer.seek(0)
        else:
            self._buffer.seek(0)
        if self._was_truncated:
            self._fileobj.truncate(0)
            self._was_truncated = False
        for chunk in self._buffer_chunks():
            self._fileobj.write(chunk)

    def flush(self):
        # flush the buffer; we only write to the underlying file on close
        self._buffer.flush()

    def close(self):
        if self.closed:
            return
        if self._check_mode("w"):
            self._write_out_buffer()
        super(Buffer, self).close()
        self._buffer.close()

    def _read(self, sizehint=-1):
        #  First return any data available from the buffer.
        #  Since we don't flush the buffer after every write, certain OSes
        #  (guess which!) will happily read junk data from the end of it.
        #  Instead, we explicitly read only up to self._in_pos.
        if not self._in_eof:
            buffered_size = self._in_pos - self._buffer.tell()
            if sizehint >= 0:
                buffered_size = min(sizehint, buffered_size)
        else:
            buffered_size = sizehint
        data = self._buffer.read(buffered_size)
        if data != "":
            return data
        # Then look for more data in the underlying file
        if self._in_eof:
            return None
        data = self._fileobj.read(sizehint)
        self._in_pos += len(data)
        self._buffer.write(data)
        if sizehint < 0 or len(data) < sizehint:
            self._in_eof = True
            self._buffer.flush()
        return data

    def _write(self, data, flushing=False):
        self._buffer.write(data)
        if self._check_mode("r") and not self._in_eof:
            diff = self._buffer.tell() - self._in_pos
            if diff > 0:
                junk = self._fileobj.read(diff)
                self._in_pos += len(junk)
                if len(junk) < diff:
                    self._in_eof = True
                    self._buffer.flush()

    def _seek(self, offset, whence):
        # Ensure we've read enough to simply do the seek on the buffer
        if self._check_mode("r") and not self._in_eof:
#.........这里部分代码省略.........