Python Dropbox.files_upload_session_finish方法代码示例

# 需要导入模块: from dropbox import Dropbox [as 别名]
# 或者: from dropbox.Dropbox import files_upload_session_finish [as 别名]
class DropboxWriter(FilebaseBaseWriter):
    """
    Writes items to dropbox folder.
    options available

        - access_token (str)
            Oauth access token for Dropbox api.

        - filebase (str)
            Base path to store the items in the share.

    """
    supported_options = {
        'access_token': {'type': six.string_types, 'env_fallback': 'EXPORTERS_DROPBOXWRITER_TOKEN'},
    }

    def __init__(self, *args, **kw):
        from dropbox import Dropbox
        super(DropboxWriter, self).__init__(*args, **kw)
        access_token = self.read_option('access_token')
        self.set_metadata('files_counter', Counter())
        self.client = Dropbox(access_token)

    def write(self, dump_path, group_key=None, file_name=False):
        if group_key is None:
            group_key = []
        self._write_file(dump_path, group_key, file_name)

    @retry_long
    def _upload_file(self, input_file, filepath):
        from dropbox import files
        session_id = self.client.files_upload_session_start('')
        current_offset = 0
        while True:
            data = input_file.read(2**20)
            if not data:
                break
            self.client.files_upload_session_append(data, session_id.session_id, current_offset)
            current_offset += len(data)
        cursor = files.UploadSessionCursor(session_id.session_id, current_offset)
        self.client.files_upload_session_finish(
            '', cursor, files.CommitInfo(path='{}'.format(filepath)))

    def _write_file(self, dump_path, group_key, file_name=None):
        filebase_path, file_name = self.create_filebase_name(group_key, file_name=file_name)
        with open(dump_path, 'r') as f:
            self._upload_file(f, '{}/{}'.format(filebase_path, file_name))
        self.get_metadata('files_counter')[filebase_path] += 1

    def get_file_suffix(self, path, prefix):
        number_of_keys = self.get_metadata('files_counter').get(path, 0)
        suffix = '{}'.format(str(number_of_keys))
        return suffix

# 需要导入模块: from dropbox import Dropbox [as 别名]
# 或者: from dropbox.Dropbox import files_upload_session_finish [as 别名]
def upload_to_dropbox(access_token, dropbox_path, file_path, progress_callback=None):
    dbx = Dropbox(access_token)
    with open(file_path, 'rb') as file:
        chunk = file.read(CHUNK_SIZE)
        offset = len(chunk)

        upload_session = dbx.files_upload_session_start(chunk)
        progress_callback and progress_callback(offset)

        while True:
            chunk = file.read(CHUNK_SIZE)
            if not chunk:
                break
            dbx.files_upload_session_append_v2(
                chunk,
                UploadSessionCursor(
                    upload_session.session_id,
                    offset,
                ),
            )
            offset += len(chunk)
            progress_callback and progress_callback(offset)

        file_metadata = dbx.files_upload_session_finish(
            b'',
            UploadSessionCursor(
                upload_session.session_id,
                offset=offset,
            ),
            CommitInfo(
                dropbox_path,
                # When writing the file it won't overwrite an existing file, just add
                # another file like "filename (2).txt"
                WriteMode('add'),
            ),
        )
        progress_callback and progress_callback(offset)
        return file_metadata.path_display

# 需要导入模块: from dropbox import Dropbox [as 别名]
# 或者: from dropbox.Dropbox import files_upload_session_finish [as 别名]

#.........这里部分代码省略.........
            log.Debug('dpbx,files_upload_session_start([%d bytes]), total: %d' % (len(buf), file_size))
            upload_sid = self.api_client.files_upload_session_start(buf)
            log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid)
            upload_cursor = UploadSessionCursor(upload_sid.session_id, f.tell())
            commit_info = CommitInfo(remote_path, mode=WriteMode.overwrite, autorename=False, client_modified=None, mute=True)
            res_metadata = None
            progress.report_transfer(f.tell(), file_size)

            requested_offset = None
            current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
            retry_number = globals.num_retries

            # We're doing our own error handling and retrying logic because
            # we can benefit from Dpbx chunked upload and retry only failed chunk
            while (f.tell() < file_size) or not res_metadata:
                try:
                    if requested_offset is not None:
                        upload_cursor.offset = requested_offset

                    if f.tell() != upload_cursor.offset:
                        f.seek(upload_cursor.offset)
                    buf = f.read(current_chunk_size)

                    # reset temporary status variables
                    requested_offset = None
                    current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
                    retry_number = globals.num_retries

                    if len(buf) != 0:
                        log.Debug('dpbx,files_upload_sesssion_append([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset))
                        self.api_client.files_upload_session_append(buf, upload_cursor.session_id, upload_cursor.offset)
                    else:
                        log.Debug('dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset))
                        res_metadata = self.api_client.files_upload_session_finish(buf, upload_cursor, commit_info)

                    upload_cursor.offset = f.tell()
                    log.Debug('progress: %d of %d' % (upload_cursor.offset, file_size))
                    progress.report_transfer(upload_cursor.offset, file_size)
                except ApiError as e:
                    error = e.error
                    if isinstance(error, UploadSessionLookupError) and error.is_incorrect_offset():
                        # Server reports that we should send another chunk. Most likely this is caused by
                        # network error during previous upload attempt. In such case we'll get expected offset
                        # from server and it's enough to just seek() and retry again
                        new_offset = error.get_incorrect_offset().correct_offset
                        log.Debug('dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)' % (upload_cursor.offset, new_offset))
                        if requested_offset is not None:
                            # chunk failed even after seek attempt. Something strange and no safe way to recover
                            raise BackendException("dpbx: unable to chunk upload")
                        else:
                            # will seek and retry
                            requested_offset = new_offset
                        continue
                    raise
                except ConnectionError as e:
                    log.Debug('dpbx,files_upload_session_append: %s' % e)

                    retry_number -= 1
                    if retry_number == 0:
                        raise

                    # We don't know for sure, was partial upload successfull or not. So it's better to retry smaller amount to avoid extra reupload
                    log.Info('dpbx: sleeping a bit before chunk retry')
                    time.sleep(30)
                    current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5
                    requested_offset = None