Python SwiftService.download方法代码示例

# 需要导入模块: from swiftclient.service import SwiftService [as 别名]
# 或者: from swiftclient.service.SwiftService import download [as 别名]
class SwiftClient(object):
    """Client for Swift object/blob store of Openstack

    See http://swift.openstack.org

    Swift requires environment variables (OS_*) for the authentication and configuration"""

    def __init__(self, container, prefix=''):
        self.container = container
        self.prefix = prefix
        self.client = SwiftService()

    def download(self, source, target):
        objects = [self.prefix + '/' + source]
        options = {'out_file': target}
        return list(self.client.download(self.container, objects, options))

    def upload(self, source, target):
        object_name = self.prefix + '/' + target
        objects = [SwiftUploadObject(source, object_name=object_name)]
        return list(self.client.upload(self.container, objects))

    def ls(self, path):
        fpath = self.prefix + '/' + path + '/'
        clisting = self.client.list(self.container, {'prefix': fpath})
        listing = list(clisting)[0]['listing']
        result = [d['name'].replace(fpath, '') for d in listing]
        return result

    def url(self, path=''):
        return self.container + '/' + self.prefix + '/' + path

# 需要导入模块: from swiftclient.service import SwiftService [as 别名]
# 或者: from swiftclient.service.SwiftService import download [as 别名]
def getObjsBackend(objs, backend, config):

    if(backend == 'hdfs'):

        client = Client(socket.gethostname(), config['HADOOP_RPC_PORT'], use_trash=False)

        for obj in objs:
                try:
                    copy_gen = client.copyToLocal([obj[0]], obj[1])
                    for copy_item in copy_gen:
                        pass
                except Exception as e:
                        print(e)
    elif(backend == 'swift'):

        options = {'os_auth_url': os.environ['OS_AUTH_URL'], 'os_username': os.environ['OS_USERNAME'], 'os_password': os.environ['OS_PASSWORD'], 'os_tenant_id': os.environ['OS_TENANT_ID'], 'os_tenant_name': os.environ['OS_TENANT_NAME']}
        swiftService = SwiftService(options=options)

        for obj in objs:

            # Create the containers which are used in this application for Object Storage
            if(obj[0] == 'sqlite.db'):
                swiftService.post(container='containerFiles')
                swiftService.post(container='containerFeatures')
                swiftService.post(container='containerModules')

            out_file = obj[1]  # Get the output file location from runner
            localoptions = {'out_file': out_file}
            objects = []
            objects.append(obj[0])
            swiftDownload = swiftService.download(container='containerModules', objects=objects, options=localoptions)

            for downloaded in swiftDownload:
                if("error" in downloaded.keys()):
                    raise RuntimeError(downloaded['error'])
                # print(downloaded)

    elif(backend == 'nfs'):  # Every file is already in respective local dirs
        pass

# 需要导入模块: from swiftclient.service import SwiftService [as 别名]
# 或者: from swiftclient.service.SwiftService import download [as 别名]
def download(source, dest=None):
    """
    Download files from a SWIFT object store
    @param source_info: Source information such as the source to download from.
    @type source_info: a dictionary
    @param local_dest_dir: The local directory to store file
    @type local_dest_dir: str
    @return: True and a list of file downloaded if successful. Otherwise False.
    """

    if not dest:
        dest = tempfile.mkstemp()

    url = urlsplit(source['url'])
    _, ver, account, container, object_name = url.path.split('/', 4)

    swift_opts = {
        'os_storage_url': '{scheme}://{netloc}/{ver}/{account}'.format(
            scheme=re.sub(r'^swift\+', '', url.scheme),
            netloc=url.netloc,
            ver=ver,
            account=account)
    }
    # SwiftService knows about environment variables
    for opt in ('os_auth_url', 'os_username', 'os_password', 'os_tenant_name', 'os_storage_url'):
        if opt in source:
            swift_opts[opt] = source[opt]
    try:
        swift = SwiftService(swift_opts)
        filelist = []

        if os.path.exists(dest) and os.path.isdir(dest):
            outfilename = os.path.join(dest, os.path.basename(object_name))
        else:
            outfilename = dest

        retries = 5  # number of retries left
        backoff = 30  # wait time between retries
        backoff_inc = 30  # increase in wait time per retry

        while retries:
            filelist = []
            retries -= 1

            try:
                for result in swift.download(container, [object_name], {'out_file': outfilename}):
                    # result dict:  success
                    #    action: 'download_object'
                    #    success: True
                    #    container: ...
                    #    object: ...
                    #    path: ....
                    #    pseudodir: ...
                    #    start_time, finish_time, headers_receipt, auth_end_time,
                    #    read_length, attempts, response_dict
                    # result dict: error
                    #    action: 'download_object'
                    #    success: False
                    #    error: ...
                    #    traceback: ...
                    #    container, object, error_timestamp, response_dict, path
                    #    psudodir, attempts
                    if not result['success']:
                        raise Exception(
                            'Download from selfelfwift {container}/{object} to {out_file} failed with {error}'.format(out_file=outfilename, **result))
                    outfile = {'url': outfilename,
                               'name': os.path.basename(object_name),
                               'content_type': result['response_dict']['headers'].get('content-type', 'application/octet-stream')}
                    filelist.append(outfile)
                # no exception we can continue
                retries = 0
            except Exception as e:
                if not retries:
                    # reraise if no retries left
                    raise
                LOG.warn("Download from Swift failed: %s - %d retries left", e, retries)
                time.sleep(backoff)
                backoff += backoff_inc
                backoff_inc += 30
        return filelist
    except Exception as e:
        LOG.error("Download from Swift failed: %s", e, exc_info=True)
        raise