Python Ring.get_nodes方法代码示例

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_nodes [as 别名]
 def __call__(self, env, start_response):
     req = Request(env)
     if env.get('REQUEST_METHOD') == "PUT" and env.get("HTTP_X_OBJECT_META_LXC_DEPLOY"):
         ring = Ring(self.object_ring_path)
         raw_path = env.get("RAW_PATH_INFO").split("/")
         node_data = ring.get_nodes(raw_path[2],raw_path[3],raw_path[4])
         deploy_host = node_data[1][0]["ip"]
         req.headers["X-Object-Meta-LXC-HOST"] = deploy_host
         req.headers["REMOTE_USER"] = raw_path[2]
     return self.app(env, start_response)

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_nodes [as 别名]
 def create_account(act):
     ts = utils.normalize_timestamp(time())
     account_ring = Ring(_testdir, ring_name='account')
     partition, nodes = account_ring.get_nodes(act)
     for node in nodes:
         # Note: we are just using the http_connect method in the object
         # controller here to talk to the account server nodes.
         conn = swift.proxy.controllers.obj.http_connect(
             node['ip'], node['port'], node['device'], partition, 'PUT',
             '/' + act, {'X-Timestamp': ts, 'x-trans-id': act})
         resp = conn.getresponse()
         assert(resp.status == 201)

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_nodes [as 别名]
    def _delete_or_save_lifecycle(self, method, lifecycle=None):
        path = '/.s3_bucket_lifecycle/%s/%s' % (self.account, self.container)
        oring = Ring('/etc/swift', ring_name='object')
        cring = Ring('/etc/swift', ring_name='container')
        part, nodes = oring.get_nodes('.s3_bucket_lifecycle', self.account,
                                      self.container)
        cpart, cnodes = cring.get_nodes('.s3_bucket_lifecycle', self.account)
        now_ts = normalize_timestamp(time.time())

        i = 0
        for node in nodes:
            ip = node['ip']
            port = node['port']
            dev = node['device']
            headers = dict()
            headers['user-agent'] = 'lifecycle-uploader'
            headers['X-Timestamp'] = now_ts
            headers['referer'] = 'lifecycle-uploader'
            headers['X-Container-Partition'] = cpart
            headers['X-Container-Host'] = '%(ip)s:%(port)s' % cnodes[i]
            headers['X-Container-Device'] = cnodes[i]['device']

            if lifecycle:
                headers['content-length'] = len(lifecycle)
                headers['etags'] = self._compute_md5(lifecycle)
                headers['content-type'] = 'text/plain'

            conn = http_connect(ip, port, dev, part, method, path,
                                headers)

            if method == 'PUT':
                conn.send(lifecycle)

            response = conn.getresponse()
            i += 1
        return response

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_nodes [as 别名]
class ObjectEndpoint(object):

    def __init__(self, app, conf):
        self.app = app
        self.logger = get_logger(conf, log_route='object_endpoint')
        swift_dir = conf.get('swift_dir', '/etc/swift')
        self.object_ring = Ring(swift_dir, ring_name='object')

    def __call__(self, env, start_response):
        request = Request(env)

        url_prefix = '/object_endpoint/'

        if request.path.startswith(url_prefix):

            if request.method != 'GET':
                raise HTTPMethodNotAllowed()

            aco = split_path(request.path[len(url_prefix) - 1:], 1, 3, True)
            account = aco[0]
            container = aco[1]
            obj = aco[2]
            if obj.endswith('/'):
                obj = obj[:-1]

            object_partition, objects = self.object_ring.get_nodes(
                account, container, obj)

            endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
                                '{account}/{container}/{obj}'
            endpoints = []
            for element in objects:
                endpoint = endpoint_template.format(ip=element['ip'],
                                                    port=element['port'],
                                                    device=element['device'],
                                                    partition=object_partition,
                                                    account=account,
                                                    container=container,
                                                    obj=obj)
                endpoints.append(endpoint)

            start_response('200 OK', {})
            return json.dumps(endpoints)

        return self.app(env, start_response)

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_nodes [as 别名]
 def create_account(act):
     ts = utils.normalize_timestamp(time())
     account_ring = Ring(_testdir, ring_name="account")
     partition, nodes = account_ring.get_nodes(act)
     for node in nodes:
         # Note: we are just using the http_connect method in the object
         # controller here to talk to the account server nodes.
         conn = swift.proxy.controllers.obj.http_connect(
             node["ip"],
             node["port"],
             node["device"],
             partition,
             "PUT",
             "/" + act,
             {"X-Timestamp": ts, "x-trans-id": act},
         )
         resp = conn.getresponse()
         assert resp.status == 201

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_nodes [as 别名]
def get_container_list(account):
    #Require a account eg. AUTH_ss
    #Return a list of containers within this account
    account_ring = Ring(swift_dir, ring_name="account")
    container_ring = Ring(swift_dir, ring_name="container")
    object_ring = Ring(swift_dir, ring_name="object")
    part, nodes = account_ring.get_nodes(account)

    URL="http://%s:%s/%s/%s/%s" % (nodes[0]['ip'], nodes[0]['port'], nodes[0]['device'],
                                   part, account)
    r = requests.get(URL)
    if r.status_code == 404:
        logger.warning("Account not existing yet")
    content = str(r.text)
    req = urllib2.Request(URL)
    container_list_hash = hashlib.md5(content).hexdigest()
    content = content.split("\n")
    content.remove('')
    return content, container_list_hash

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_nodes [as 别名]
    def _get_db_info(self, account, container, number):
        server_type = 'container'
        obj_conf = self.configs['%s-server' % server_type]
        config_path = obj_conf[number]
        options = utils.readconf(config_path, 'app:container-server')
        root = options.get('devices')

        swift_dir = options.get('swift_dir', '/etc/swift')
        ring = Ring(swift_dir, ring_name=server_type)
        part, nodes = ring.get_nodes(account, container)
        for node in nodes:
            # assumes one to one mapping
            if node['port'] == int(options.get('bind_port')):
                device = node['device']
                break
        else:
            return None

        path_hash = utils.hash_path(account, container)
        _dir = utils.storage_directory('%ss' % server_type, part, path_hash)
        db_dir = os.path.join(root, device, _dir)
        db_file = os.path.join(db_dir, '%s.db' % path_hash)
        db = ContainerBroker(db_file)
        return db.get_info()

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_nodes [as 别名]
    def _test_ondisk_data_after_write_with_crypto(self, policy_name):
        policy = storage_policy.POLICIES.get_by_name(policy_name)
        self._create_container(self.proxy_app, policy_name=policy_name)
        self._put_object(self.crypto_app, self.plaintext)
        self._post_object(self.crypto_app)

        # Verify container listing etag is encrypted by direct GET to container
        # server. We can use any server for all nodes since they all share same
        # devices dir.
        cont_server = self._test_context['test_servers'][3]
        cont_ring = Ring(self._test_context['testdir'], ring_name='container')
        part, nodes = cont_ring.get_nodes('a', self.container_name)
        for node in nodes:
            req = Request.blank('/%s/%s/a/%s'
                                % (node['device'], part, self.container_name),
                                method='GET', query_string='format=json')
            resp = req.get_response(cont_server)
            listing = json.loads(resp.body)
            # sanity checks...
            self.assertEqual(1, len(listing))
            self.assertEqual('o', listing[0]['name'])
            self.assertEqual('application/test', listing[0]['content_type'])
            # verify encrypted etag value
            parts = listing[0]['hash'].rsplit(';', 1)
            crypto_meta_param = parts[1].strip()
            crypto_meta = crypto_meta_param[len('swift_meta='):]
            listing_etag_iv = load_crypto_meta(crypto_meta)['iv']
            exp_enc_listing_etag = base64.b64encode(
                encrypt(self.plaintext_etag,
                        self.km.create_key('/a/%s' % self.container_name),
                        listing_etag_iv))
            self.assertEqual(exp_enc_listing_etag, parts[0])

        # Verify diskfile data and metadata is encrypted
        ring_object = self.proxy_app.get_object_ring(int(policy))
        partition, nodes = ring_object.get_nodes('a', self.container_name, 'o')
        conf = {'devices': self._test_context["testdir"],
                'mount_check': 'false'}
        df_mgr = diskfile.DiskFileRouter(conf, FakeLogger())[policy]
        ondisk_data = []
        exp_enc_body = None
        for node_index, node in enumerate(nodes):
            df = df_mgr.get_diskfile(node['device'], partition,
                                     'a', self.container_name, 'o',
                                     policy=policy)
            with df.open():
                meta = df.get_metadata()
                contents = ''.join(df.reader())
                metadata = dict((k.lower(), v) for k, v in meta.items())
                # verify on disk data - body
                body_iv = load_crypto_meta(
                    metadata['x-object-sysmeta-crypto-body-meta'])['iv']
                body_key_meta = load_crypto_meta(
                    metadata['x-object-sysmeta-crypto-body-meta'])['body_key']
                obj_key = self.km.create_key('/a/%s/o' % self.container_name)
                body_key = Crypto().unwrap_key(obj_key, body_key_meta)
                exp_enc_body = encrypt(self.plaintext, body_key, body_iv)
                ondisk_data.append((node, contents))

                # verify on disk user metadata
                enc_val, meta = metadata[
                    'x-object-transient-sysmeta-crypto-meta-fruit'].split(';')
                meta = meta.strip()[len('swift_meta='):]
                metadata_iv = load_crypto_meta(meta)['iv']
                exp_enc_meta = base64.b64encode(encrypt('Kiwi', obj_key,
                                                        metadata_iv))
                self.assertEqual(exp_enc_meta, enc_val)
                self.assertNotIn('x-object-meta-fruit', metadata)

                self.assertIn(
                    'x-object-transient-sysmeta-crypto-meta', metadata)
                meta = load_crypto_meta(
                    metadata['x-object-transient-sysmeta-crypto-meta'])
                self.assertIn('key_id', meta)
                self.assertIn('path', meta['key_id'])
                self.assertEqual(
                    '/a/%s/%s' % (self.container_name, self.object_name),
                    meta['key_id']['path'])
                self.assertIn('v', meta['key_id'])
                self.assertEqual('1', meta['key_id']['v'])
                self.assertIn('cipher', meta)
                self.assertEqual(Crypto.cipher, meta['cipher'])

                # verify etag
                actual_enc_etag, _junk, actual_etag_meta = metadata[
                    'x-object-sysmeta-crypto-etag'].partition('; swift_meta=')
                etag_iv = load_crypto_meta(actual_etag_meta)['iv']
                exp_enc_etag = base64.b64encode(encrypt(self.plaintext_etag,
                                                        obj_key, etag_iv))
                self.assertEqual(exp_enc_etag, actual_enc_etag)

                # verify etag hmac
                exp_etag_mac = hmac.new(
                    obj_key, self.plaintext_etag, digestmod=hashlib.sha256)
                exp_etag_mac = base64.b64encode(exp_etag_mac.digest())
                self.assertEqual(exp_etag_mac,
                                 metadata['x-object-sysmeta-crypto-etag-mac'])

                # verify etag override for container updates
                override = 'x-object-sysmeta-container-update-override-etag'
#.........这里部分代码省略.........

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_nodes [as 别名]

#.........这里部分代码省略.........
        clean_path = request.path[len(self.endpoints_path) - 1:]
        # try to peel off version
        try:
            raw_version, rest = split_path(clean_path, 1, 2, True)
        except ValueError:
            raise ValueError('No account specified')
        try:
            version = self._parse_version(raw_version)
        except ValueError:
            if raw_version.startswith('v') and '_' not in raw_version:
                # looks more like a invalid version than an account
                raise
            # probably no version specified, but if the client really
            # said /endpoints/v_3/account they'll probably be sorta
            # confused by the useless response and lack of error.
            version = self.default_response_version
            rest = clean_path
        else:
            rest = '/' + rest if rest else '/'
        try:
            account, container, obj = split_path(rest, 1, 3, True)
        except ValueError:
            raise ValueError('No account specified')
        return version, account, container, obj

    def v1_format_response(self, req, endpoints, **kwargs):
        return Response(json.dumps(endpoints),
                        content_type='application/json')

    def v2_format_response(self, req, endpoints, storage_policy_index,
                           **kwargs):
        resp = {
            'endpoints': endpoints,
            'headers': {},
        }
        if storage_policy_index is not None:
            resp['headers'][
                'X-Backend-Storage-Policy-Index'] = str(storage_policy_index)
        return Response(json.dumps(resp),
                        content_type='application/json')

    def __call__(self, env, start_response):
        request = Request(env)
        if not request.path.startswith(self.endpoints_path):
            return self.app(env, start_response)

        if request.method != 'GET':
            return HTTPMethodNotAllowed(
                req=request, headers={"Allow": "GET"})(env, start_response)

        try:
            version, account, container, obj = self._parse_path(request)
        except ValueError as err:
            return HTTPBadRequest(str(err))(env, start_response)

        if account is not None:
            account = unquote(account)
        if container is not None:
            container = unquote(container)
        if obj is not None:
            obj = unquote(obj)

        storage_policy_index = None
        if obj is not None:
            container_info = get_container_info(
                {'PATH_INFO': '/v1/%s/%s' % (account, container)},
                self.app, swift_source='LE')
            storage_policy_index = container_info['storage_policy']
            obj_ring = self.get_object_ring(storage_policy_index)
            partition, nodes = obj_ring.get_nodes(
                account, container, obj)
            endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
                                '{account}/{container}/{obj}'
        elif container is not None:
            partition, nodes = self.container_ring.get_nodes(
                account, container)
            endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
                                '{account}/{container}'
        else:
            partition, nodes = self.account_ring.get_nodes(
                account)
            endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
                                '{account}'

        endpoints = []
        for node in nodes:
            endpoint = endpoint_template.format(
                ip=node['ip'],
                port=node['port'],
                device=node['device'],
                partition=partition,
                account=quote(account),
                container=quote(container or ''),
                obj=quote(obj or ''))
            endpoints.append(endpoint)

        resp = self.response_map[version](
            request, endpoints=endpoints,
            storage_policy_index=storage_policy_index)
        return resp(env, start_response)

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_nodes [as 别名]
from swift.common.ring import Ring

if __name__ == '__main__':
    # example path for sample object
    # update this to and existing account/container/object 
    # in your environment
    account = 'AUTH_9fbaa44c45ab4902a46110fd90629a79'
    container = 'testing'
    obj = 'testing.pem'

    ring = Ring('.', ring_name='object')
    part, nodes =  ring.get_nodes(account, container, obj)
    print 'nodes: '
    for n in nodes:
        print 'node: ', n

    print 'part = ', part
    morenodes = ring.get_more_nodes(part)
    print 'more nodes:'
    for n in morenodes:
        print 'node: ', n

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_nodes [as 别名]
class FileMover(object):
    def __init__(self, options, *_args, **_kwargs):
        self.ring = Ring(options.ring)
        self.path = options.path
        self.options = options

    def _get_acc_cont_obj(self, filename):
        """ Returns account, container, object from XFS object metadata """

        obj_fd = open(filename)
        metadata = ''
        key = 0
        try:
            while True:
                metadata += xattr.getxattr(
                    obj_fd, '%s%s' % ("user.swift.metadata", (key or '')))
                key += 1
        except IOError:
            pass
        obj_fd.close()
        object_name = pickle.loads(metadata).get('name')
        account = object_name.split('/')[1]
        container = object_name.split('/')[2]
        obj = '/'.join(object_name.split('/')[3:])

        return {'account': account,
                'container': container,
                'object': obj}

    def start(self):
        for root, _dirs, files in os.walk(self.path):
            if "quarantined" in root:
                continue
            for filename in files:
                fullname = os.path.join(root, filename)
                if (self.options.move_object_files is True and
                        fullname.split('.')[-1] in ["data", "ts"]):
                    self._move_file(fullname, "objects")

                if (self.options.move_container_dbs is True and
                        fullname.split('.')[-1] in ["db"] and
                        "containers" in fullname):
                    self._move_file(fullname, "containers")

                if (self.options.move_account_dbs is True and
                        fullname.split('.')[-1] in ["db"] and
                        "accounts" in fullname):
                    self._move_file(fullname, "accounts")

    def _move_file(self, filename, filetype):
        if filetype == 'accounts':
            broker = AccountBroker(filename)
            info = broker.get_info()
        elif filetype == 'containers':
            broker = ContainerBroker(filename)
            info = broker.get_info()
        elif filetype == 'objects':
            info = self._get_acc_cont_obj(filename)
        else:
            raise Exception

        acc = info.get('account')
        cont = info.get('container')
        obj = info.get('object')

        partition, _nodes = self.ring.get_nodes(acc, cont, obj)

        # replace the old partition value with the new one
        # old name like '/a/b/objects/123/c/d'
        # new name like '/a/b/objects/456/c/d'
        filename_parts = filename.split('/')
        part_pos = filename_parts.index(filetype)
        filename_parts[part_pos+1] = str(partition)
        newname = '/'.join(filename_parts)

        dst_dir = os.path.dirname(newname)
        try:
            os.makedirs(dst_dir)
            logging.info("mkdir %s" % dst_dir)
        except OSError as ex:
            logging.info("mkdir %s failed: %s" % (dst_dir, ex))

        try:
            os.rename(filename, newname)
            logging.info("moved %s -> %s" % (filename, newname))
        except OSError as ex:
            logging.warning("FAILED TO MOVE %s -> %s" % (filename, newname))

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_nodes [as 别名]

#.........这里部分代码省略.........
                bytes_sents[bill_type] = bytes_sents.get(bill_type,
                                                         0) + int(bytes_st)
                self.report_objects += 1

            for o in objs_to_delete:
                self.swift.delete_object(self.sample_account, container, o)

            for bill_type, bt_rv in bytes_recvs.items():
                t_object = 'transfer/%d/%d/%d_%d_%d' % (ts, bill_type, bt_rv,
                                                        bytes_sents[bill_type],
                                                        self.report_objects)
                self._hidden_update(tenant_id, t_object)
        except (Exception, Timeout) as err:
            self.logger.increment('errors')
            self.logger.exception(
                _('Exception while aggregating sample %s %s') %
                (container, str(err)))

        self.logger.timing_since('timing', start_time)
        self.report()

    def account_info(self, tenant_id, timestamp):
        path = '/v1/%s/%s?prefix=usage/%d&limit=1' % (self.aggregate_account,
                                                      tenant_id, timestamp)
        resp = self.swift.make_request('GET', path, {}, (2,))
        if len(resp.body) == 0:
            return 0, 0, 0
        usages = resp.body.split('/', 2)[2].rstrip()
        cont_cnt, obj_cnt, bt_used = usages.split('_')
        return int(cont_cnt), int(obj_cnt), int(bt_used)

    def _hidden_update(self, container, obj, method='PUT'):
        hidden_path = '/%s/%s/%s' % (self.aggregate_account, container, obj)
        part, nodes = self.container_ring.get_nodes(self.aggregate_account,
                                                    container)
        for node in nodes:
            ip = node['ip']
            port = node['port']
            dev = node['device']
            action_headers = dict()
            action_headers['user-agent'] = 'aggregator'
            action_headers['X-Timestamp'] = normalize_timestamp(time())
            action_headers['referer'] = 'aggregator-daemon'
            action_headers['x-size'] = '0'
            action_headers['x-content-type'] = "text/plain"
            action_headers['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e'

            conn = http_connect(ip, port, dev, part, method, hidden_path,
                                action_headers)
            response = conn.getresponse()
            response.read()

    def fillup_lossed_usage_data(self, tenants):
        now = (float(time()) // self.sample_rate) * self.sample_rate
        path = '/v1/%s/%s?prefix=usage/%d&limit=1'

        for t in tenants:
            last = self.last_chk
            cont_cnt = obj_cnt = bt_used = -1
            while last <= now:
                p = path % (self.aggregate_account, t, last)
                resp = self.swift.make_request('GET', p, {}, (2,))
                if len(resp.body) != 0:
                    usages = resp.body.split('/', 2)[2].rstrip()
                    c, o, bt = usages.split('_')
                    cont_cnt = int(c)

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_nodes [as 别名]
class TransitionMiddleware(object):
    def __init__(self, app, conf, *args, **kwargs):
        self.app = app
        self.conf = conf
        self.logger = get_logger(self.conf, log_route='transition')
        self.container_ring = Ring('/etc/swift', ring_name='container')
        self.glacier_account_prefix = '.glacier_'
        self.temp_path = conf.get('temp_path', '/var/cache/s3/')

    def _init_glacier(self):
        con = Layer2(region_name='ap-northeast-1')
        return con.create_vault('swift-s3-transition')

    def transition(self, env):
        # GET Object body
        req = Request(copy(env))
        req.method = 'GET'
        resp = req.get_response(self.app)

        obj_body = resp.body

        # Glacier로 업로드
        tmpfile = self.save_to_tempfile(obj_body)
        try:
            glacier = self._init_glacier()
            archive_id = glacier.upload_archive(tmpfile)
            glacier_obj = make_glacier_hidden_object_name(self.obj, archive_id)
        except Exception as e:
            return Response(status=HTTP_INTERNAL_SERVER_ERROR, body=e.message)
        finally:
            self.delete_tempfile(tmpfile)

        # Object를 0KB로 만들기
        req = Request(copy(env))
        req.headers[GLACIER_FLAG_META] = True
        resp = req.get_response(self.app)

        # Glacier Hidden account에 기록
        glacier_account = self.glacier_account_prefix + self.account
        part, nodes = self.container_ring.get_nodes(glacier_account,
                                                    self.container)
        hidden_path = '/%s/%s/%s' % (glacier_account, self.container,
                                     glacier_obj)
        for node in nodes:
            ip = node['ip']
            port = node['port']
            dev = node['device']
            headers = dict()
            headers['user-agent'] = 'transition-middleware'
            headers['X-Timestamp'] = normalize_timestamp(time.time())
            headers['referer'] = req.as_referer()
            headers['x-size'] = '0'
            headers['x-content-type'] = 'text/plain'
            headers['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e'

            conn = http_connect(ip, port, dev, part, 'PUT', hidden_path,
                                headers)
            conn.getresponse().read()
        return Response(status=HTTP_NO_CONTENT)

    def save_to_tempfile(self, data):
        tmp_path = None
        try:
            with tempfile.NamedTemporaryFile(bufsize=0, delete=False,
                                             dir=self.temp_path) as temp:
                temp.write(data)
                temp.flush()
                tmp_path = temp.name
        except Exception as e:
            self.logger.error(e)
        return tmp_path

    def delete_tempfile(self, tmppath):
        os.remove(tmppath)

    def __call__(self, env, start_response):
        req = Request(env)
        method = req.method
        self.version, self.account, self.container, self.obj = split_path(
            req.path, 0, 4, True)
        if not self.obj:
            return self.app(env, start_response)

        if method == 'POST' and \
           'X-S3-Object-Transition' in req.headers:
            return self.transition(env)(env, start_response)

        return self.app(env, start_response)

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_nodes [as 别名]

#.........这里部分代码省略.........
            return None

        return get_glacier_key_from_hidden_object(glacier_obj)

    def check_object_restored(self, restoring_object):
        actual_obj = get_glacier_objname_from_hidden_object(restoring_object)
        jobId = get_glacier_key_from_hidden_object(restoring_object)
        try:
            path = '/v1/%s' % actual_obj
            resp = self.swift.make_request('GET', path, {}, (2, 4,))
            if resp.status_int == 404:
                raise Exception('Object Not Found: %s' % actual_obj)

            job = self.glacier.get_job(job_id=jobId)
            if not job.completed:
                return
            self.complete_restore(actual_obj, job)
        except Exception as e:
            # Job ID가 만료될 경우 다시 restore 를 시도한다.
            if not e.message.startswith('Object Not Found:'):
                self.start_object_restoring(actual_obj)
            self.logger.info(e)

        self.swift.delete_object(self.restoring_object_account,
                                 self.restoring_container, restoring_object)

    def complete_restore(self, actual_obj, job):
        tmppath = tempfile.NamedTemporaryFile(bufsize=0, delete=False,
                                              dir=self.glacier_tmpdir).name
        try:
            job.download_to_file(filename=tmppath)

            prefix = 'X-Object-Meta'
            a, c, o = actual_obj.split('/', 2)
            metadata = self.swift.get_object_metadata(a, c, o,
                                                      metadata_prefix=prefix)
            metadata = {'X-Object-Meta' + key: value for key, value in metadata
            .iteritems()}
            days = int(metadata['X-Object-Meta-s3-restore-expire-days'])
            exp_time = normalize_delete_at_timestamp(calc_nextDay(time()) +
                                                     (days - 1) * 86400)

            # send restored object to proxy server
            path = '/v1/%s' % actual_obj
            metadata['X-Object-Meta-S3-Restored'] = True
            exp_date = strftime("%a, %d %b %Y %H:%M:%S GMT",
                                gmtime(float(exp_time)))

            metadata['X-Object-Meta-s3-restore'] = 'ongoing-request="false", ' \
                                                   'expiry-date="%s"' % exp_date
            metadata['Content-Length'] = os.path.getsize(tmppath)
            del metadata['X-Object-Meta-s3-restore-expire-days']

            obj_body = open(tmppath, 'r')
            self.swift.make_request('PUT', path, metadata, (2,),
                                    body_file=obj_body)

            # Add to .s3_expiring_restored_objects
            self.update_action_hidden(self.expiring_restored_account,
                                      exp_time, actual_obj)
            obj_body.close()
            self.logger.increment('done')
        except UnexpectedResponse as e:
            if e.resp.status_int == 404:
                self.logger.error('Restoring object not found - %s' %
                                  actual_obj)
        except Exception as e:
            self.logger.increment('errors')
            self.logger.debug(e)
        finally:
            os.remove(tmppath)

    def compute_obj_md5(self, obj):
        etag = hashlib.md5()
        etag.update(obj)
        etag = etag.hexdigest()
        return etag

    def update_action_hidden(self, account, container, obj, metadata=None):
        hidden_path = '/%s/%s/%s' % (account, container, obj)
        part, nodes = self.container_ring.get_nodes(account, container)
        for node in nodes:
            ip = node['ip']
            port = node['port']
            dev = node['device']
            action_headers = dict()
            action_headers['user-agent'] = 'restore-daemon'
            action_headers['X-Timestamp'] = normalize_timestamp(time())
            action_headers['referer'] = 'restore-daemon'
            action_headers['x-size'] = '0'
            action_headers['x-content-type'] = "text/plain"
            action_headers['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e'

            if metadata:
                action_headers.update(metadata)

            conn = http_connect(ip, port, dev, part, 'PUT', hidden_path,
                                action_headers)
            response = conn.getresponse()
            response.read()

# 需要导入模块: from swift.common.ring import Ring [as 别名]
# 或者: from swift.common.ring.Ring import get_nodes [as 别名]

#.........这里部分代码省略.........

        req = Request(env)
        if self.check_api_call(env):
            return self.GET(req)(env, start_response)

        try:
            version, account, container, obj = req.split_path(2, 4, True)
        except ValueError:
            return self.app(env, start_response)

        remote_user = env.get('REMOTE_USER')
        if not remote_user or (isinstance(remote_user, basestring) and
                               remote_user.startswith('.wsgi')):
            self.logger.debug('### SKIP: REMOTE_USER is %s' % remote_user)
            return self.app(env, start_response)

        start_response_args = [None]
        input_proxy = InputProxy(env['wsgi.input'])
        env['wsgi.input'] = input_proxy

        def my_start_response(status, headers, exc_info=None):
            start_response_args[0] = (status, list(headers), exc_info)

        def iter_response(iterable):
            iterator = iter(iterable)
            try:
                chunk = next(iterator)
                while not chunk:
                    chunk = next(iterator)
            except StopIteration:
                chunk = ''

            if start_response_args[0]:
                start_response(*start_response_args[0])

            bytes_sent = 0
            try:
                while chunk:
                    bytes_sent += len(chunk)
                    yield chunk
                    chunk = next(iterator)
            finally:
                try:
                    self.publish_sample(env, account,
                                        input_proxy.bytes_received,
                                        bytes_sent)
                except Exception:
                    self.logger.exception('Failed to publish samples')

        try:
            iterable = self.app(env, my_start_response)
        except Exception:
            self.publish_sample(env, account, input_proxy.bytes_received, 0)
            raise
        else:
            return iter_response(iterable)

    def publish_sample(self, env, account, bytes_received, bytes_sent):
        timestamp = normalize_timestamp(time.time())
        sample_time = (float(
            timestamp) // self.sample_rate + 1) * self.sample_rate
        trans_id = env.get('swift.trans_id')
        tenant_id = env.get('HTTP_X_TENANT_ID')
        remote_addr = env.get('REMOTE_ADDR')

        # check if account information object is existed.
        if not self.swift_account(env, tenant_id):
            obj = 'account/%s' % account
            self.put_hidden_object(self.aggregate_account, tenant_id, obj)

        # recording account's storage usage data
        self.record_usage_data(env, tenant_id, account, sample_time)

        container = '%s_%s_%s' % (sample_time, tenant_id, account)

        obj = '%s/%d/%d/%s/%s' % (timestamp, bytes_received, bytes_sent,
                                  trans_id, remote_addr)
        self.put_hidden_object(self.sample_account, container, obj)

    def put_hidden_object(self, account, container, obj):
        hidden_path = '/%s/%s/%s' % (account, container, obj)
        self.logger.debug('put sample_path: %s' % hidden_path)
        part, nodes = self.container_ring.get_nodes(self.sample_account,
                                                    container)
        for node in nodes:
            ip = node['ip']
            port = node['port']
            dev = node['device']
            action_headers = dict()
            action_headers['user-agent'] = 'utilization'
            action_headers['X-Timestamp'] = normalize_timestamp(time.time())
            action_headers['referer'] = 'utilization-middleware'
            action_headers['x-size'] = '0'
            action_headers['x-content-type'] = "text/plain"
            action_headers['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e'

            conn = http_connect(ip, port, dev, part, 'PUT', hidden_path,
                                action_headers)
            response = conn.getresponse()
            response.read()