Python SwiftService.upload方法代码示例

# 需要导入模块: from swiftclient.service import SwiftService [as 别名]
# 或者: from swiftclient.service.SwiftService import upload [as 别名]
class SwiftClient(object):
    """Client for Swift object/blob store of Openstack

    See http://swift.openstack.org

    Swift requires environment variables (OS_*) for the authentication and configuration"""

    def __init__(self, container, prefix=''):
        self.container = container
        self.prefix = prefix
        self.client = SwiftService()

    def download(self, source, target):
        objects = [self.prefix + '/' + source]
        options = {'out_file': target}
        return list(self.client.download(self.container, objects, options))

    def upload(self, source, target):
        object_name = self.prefix + '/' + target
        objects = [SwiftUploadObject(source, object_name=object_name)]
        return list(self.client.upload(self.container, objects))

    def ls(self, path):
        fpath = self.prefix + '/' + path + '/'
        clisting = self.client.list(self.container, {'prefix': fpath})
        listing = list(clisting)[0]['listing']
        result = [d['name'].replace(fpath, '') for d in listing]
        return result

    def url(self, path=''):
        return self.container + '/' + self.prefix + '/' + path

# 需要导入模块: from swiftclient.service import SwiftService [as 别名]
# 或者: from swiftclient.service.SwiftService import upload [as 别名]
def saveObjsBackend(objs, backend, config):

    if(backend == 'hdfs'):
        for obj in objs:
            try:
                # obj[0] is hdfs path and obj[1] is local filesystem path
                if '.py' in obj[1]:  # If you are uploading a module do not allow it to be overwritten
                    subprocess.check_call(['hdfs', 'dfs', '-copyFromLocal', obj[1], obj[0]])
                else:  # If it is the metadata then it has to be overwritten everytime
                    subprocess.check_call(['hdfs', 'dfs', '-copyFromLocal', '-f', obj[1], obj[0]])
            except Exception as e:
                shutil.copyfile(config['BACKUP_METADATA_LOCAL_PATH'], config['METADATA_LOCAL_PATH'])
                raise RuntimeError(e)

    elif(backend == 'swift'):
        options = {'os_auth_url': os.environ['OS_AUTH_URL'], 'os_username': os.environ['OS_USERNAME'], 'os_password': os.environ['OS_PASSWORD'], 'os_tenant_id': os.environ['OS_TENANT_ID'], 'os_tenant_name': os  .environ['OS_TENANT_NAME']}
        swiftService = SwiftService(options=options)
        objects = []
        for obj in objs:
            objects.append(SwiftUploadObject(obj[1], object_name=obj[0]))

            swiftUpload = swiftService.upload(container='containerModules', objects=objects)
            for uploaded in swiftUpload:
                if("error" in uploaded.keys()):
                    shutil.copyfile(config['BACKUP_METADATA_LOCAL_PATH'], config['METADATA_LOCAL_PATH'])
                    raise RuntimeError(uploaded['error'])

    elif(backend == 'nfs'):
        for obj in objs:
            shutil.copyfile(obj[1], config['MODULES_DIR'] + obj[0])

    print('Metadata/Module changed and uploaded')

# 需要导入模块: from swiftclient.service import SwiftService [as 别名]
# 或者: from swiftclient.service.SwiftService import upload [as 别名]
def upload(source, dest):
    """
    Upload file to a remote SWIFT store
    @param source: List of local source path to upload from.
    @type source : Dicrionary
    @param dest: The destination information such as destination url to upload the file.
    @type dest: Dictionary
    @return: True if upload is successful. Otherwise False.
    """

    url = urlsplit(dest['url'])

    _, ver, account, container, object_name = url.path.split('/', 4)

    swift_opts = {
        'os_storage_url': '{scheme}://{netloc}/{ver}/{account}'.format(
            scheme=re.sub(r'^swift\+', '', url.scheme),
            netloc=url.netloc,
            ver=ver,
            account=account)
    }
    # SwiftService knows about environment variables
    for opt in ('os_auth_url', 'os_username', 'os_password', 'os_tenant_name', 'os_storage_url'):
        if opt in dest:
            swift_opts[opt] = dest[opt]
    try:
        swift = SwiftService(swift_opts)
        headers = []
        if 'content_type' in source:
            headers.append('Content-Type: {}'.format(source['content_type']))

        retries = 5  # number of retries left
        backoff = 30  # wait time between retries
        backoff_inc = 30  # increase in wait time per retry

        while retries:
            retries -= 1
            try:
                for result in swift.upload(container, [SwiftUploadObject(source['url'], object_name=object_name, options={'header': headers})]):
                    # TODO: we may get  result['action'] = 'create_container'
                    # self.assertNotIn(member, container)d result['action'] = 'upload_object';  result['path'] =
                    # source['url']
                    if not result['success']:
                        raise Exception(
                            'Upload to Swift {container}/{object_name} failed with {error}'.format(object_name=object_name, **result))
                # no exception we can continue
                retries = 0
            except Exception as e:
                if not retries:
                    # reraise if no retries left
                    raise
                LOG.warn('Upload to Swift failed: %s - %d retries left', e, retries)
                time.sleep(backoff)
                backoff += backoff_inc
                backoff_inc += 30
    except Exception as e:
        LOG.error("Upload to swift failed: %s", e, exc_info=True)
        raise

# 需要导入模块: from swiftclient.service import SwiftService [as 别名]
# 或者: from swiftclient.service.SwiftService import upload [as 别名]
 def test_upload_with_bad_segment_size(self):
     for bad in ('ten', '1234X', '100.3'):
         options = {'segment_size': bad}
         try:
             service = SwiftService(options)
             next(service.upload('c', 'o'))
             self.fail('Expected SwiftError when segment_size=%s' % bad)
         except SwiftError as exc:
             self.assertEqual('Segment size should be an integer value',
                              exc.value)

# 需要导入模块: from swiftclient.service import SwiftService [as 别名]
# 或者: from swiftclient.service.SwiftService import upload [as 别名]
class DiracStore(object):
    def __init__(self, container, topic, kafka):
        self.container = container
        self.swift = SwiftService()
        self.out = OutputManager()
        self.kafka = kafka

        auth = get_auth()
        self.url = auth[0]
        self.token = auth[1]
        
        self.topic = topic
        self.producer = KafkaProducer(bootstrap_servers=kafka)

    def send(self, resource):
        json = message_to_json(resource)
        print("sending " + json)
        self.producer.send(self.topic, json.encode('utf-8'))

    def make_resource(self, stat):
        resource = schema.Resource()
        resource.type = 'file'
        resource.name = stat['Object']
        resource.location = self.url + '/' + self.container + '/' + stat['Object']
        resource.mimeType = stat['Content Type']
        resource.size = long(stat['Content Length'])
        resource.created = stat['Last Modified']

        return resource

    def stat(self, paths):
        stat = {}
        for response in self.swift.stat(container=self.container, objects=paths):
            if response['success']:
                stat[response['object']] = {item[0]: item[1] for item in response['items']}

        return stat

    def store(self, path, source):
        isdir = os.path.isdir(source)
        base = source if isdir else os.path.dirname(source)
        sources = os.listdir(source) if isdir else [source]
        locations = [os.path.join(path, os.path.basename(file)) for file in sources]
        print(str(len(locations)) + " locations!")
        stats = self.stat(locations)
        objs = [SwiftUploadObject(os.path.join(base, os.path.basename(location)), object_name=location) for location in locations if not location in stats]
        print(str(len(objs)) + " previously unseen!")

        for response in self.swift.upload(self.container, objs):
            if response['success']:
                if 'object' in response:
                    print('uploading ' + response['object'])
                    stat = self.stat([response['object']])
                    resource = self.make_resource(stat.values()[0])
                    self.send(resource)