Python InternalClient.make_request方法代码示例

# 需要导入模块: from swift.common.internal_client import InternalClient [as 别名]
# 或者: from swift.common.internal_client.InternalClient import make_request [as 别名]
def make_swift_request(op, account, container=None, obj=None):
    """
    Makes a swift request via a local proxy (cost expensive)

    :param op: opertation (PUT, GET, DELETE, HEAD)
    :param account: swift account
    :param container: swift container
    :param obj: swift object
    :returns: swift.common.swob.Response instance
    """
    iclient = InternalClient(LOCAL_PROXY, 'SA', 1)
    path = iclient.make_path(account, container, obj)
    resp = iclient.make_request(op, path, {'PATH_INFO': path}, [200])

    return resp

# 需要导入模块: from swift.common.internal_client import InternalClient [as 别名]
# 或者: from swift.common.internal_client.InternalClient import make_request [as 别名]

#.........这里部分代码省略.........
        # Make sure there's no async_pendings anywhere. Probe tests only run
        # on single-node installs anyway, so this set should be small enough
        # that an exhaustive check doesn't take too long.
        all_obj_nodes = self.get_all_object_nodes()
        pendings_before = self.gather_async_pendings(all_obj_nodes)

        # expire the objects
        Manager(['object-expirer']).once()
        pendings_after = self.gather_async_pendings(all_obj_nodes)
        self.assertEqual(pendings_after, pendings_before)

    def test_expirer_object_should_not_be_expired(self):

        # Current object-expirer checks the correctness via x-if-delete-at
        # header that it can be deleted by expirer. If there are objects
        # either which doesn't have x-delete-at header as metadata or which
        # has different x-delete-at value from x-if-delete-at value,
        # object-expirer's delete will fail as 412 PreconditionFailed.
        # However, if some of the objects are in handoff nodes, the expirer
        # can put the tombstone with the timestamp as same as x-delete-at and
        # the object consistency will be resolved as the newer timestamp will
        # be winner (in particular, overwritten case w/o x-delete-at). This
        # test asserts such a situation that, at least, the overwriten object
        # which have larger timestamp than the original expirered date should
        # be safe.

        def put_object(headers):
            # use internal client to PUT objects so that X-Timestamp in headers
            # is effective
            headers['Content-Length'] = '0'
            path = self.client.make_path(
                self.account, self.container_name, self.object_name)
            try:
                self.client.make_request('PUT', path, headers, (2,))
            except UnexpectedResponse as e:
                self.fail(
                    'Expected 201 for PUT object but got %s' % e.resp.status)

        obj_brain = BrainSplitter(self.url, self.token, self.container_name,
                                  self.object_name, 'object', self.policy)

        # T(obj_created) < T(obj_deleted with x-delete-at) < T(obj_recreated)
        #   < T(expirer_executed)
        # Recreated obj should be appeared in any split brain case

        obj_brain.put_container()

        # T(obj_deleted with x-delete-at)
        # object-server accepts req only if X-Delete-At is later than 'now'
        # so here, T(obj_created) < T(obj_deleted with x-delete-at)
        now = time.time()
        delete_at = int(now + 2.0)
        recreate_at = delete_at + 1.0
        put_object(headers={'X-Delete-At': str(delete_at),
                            'X-Timestamp': Timestamp(now).normal})

        # some object servers stopped to make a situation that the
        # object-expirer can put tombstone in the primary nodes.
        obj_brain.stop_primary_half()

        # increment the X-Timestamp explicitly
        # (will be T(obj_deleted with x-delete-at) < T(obj_recreated))
        put_object(headers={'X-Object-Meta-Expired': 'False',
                            'X-Timestamp': Timestamp(recreate_at).normal})

        # make sure auto-created containers get in the account listing

# 需要导入模块: from swift.common.internal_client import InternalClient [as 别名]
# 或者: from swift.common.internal_client.InternalClient import make_request [as 别名]

#.........这里部分代码省略.........
            for container in containers_to_delete:
                try:
                    self.swift.delete_container(
                        self.expiring_objects_account,
                        container,
                        acceptable_statuses=(2, HTTP_NOT_FOUND, HTTP_CONFLICT))
                except (Exception, Timeout) as err:
                    self.logger.exception(
                        _('Exception while deleting container %s %s') %
                        (container, str(err)))
            self.logger.debug(_('Run end'))
            self.report(final=True)
        except (Exception, Timeout):
            self.logger.exception(_('Unhandled exception'))

    def run_forever(self, *args, **kwargs):
        """
        Executes passes forever, looking for objects to expire.

        :param args: Extra args to fulfill the Daemon interface; this daemon
                     has no additional args.
        :param kwargs: Extra keyword args to fulfill the Daemon interface; this
                       daemon has no additional keyword args.
        """
        sleep(random() * self.interval)
        while True:
            begin = time()
            try:
                self.run_once(*args, **kwargs)
            except (Exception, Timeout):
                self.logger.exception(_('Unhandled exception'))
            elapsed = time() - begin
            if elapsed < self.interval:
                sleep(random() * (self.interval - elapsed))

    def get_process_values(self, kwargs):
        """
        Gets the processes, process from the kwargs if those values exist.

        Otherwise, return processes, process set in the config file.

        :param kwargs: Keyword args passed into the run_forever(), run_once()
                       methods.  They have values specified on the command
                       line when the daemon is run.
        """
        if kwargs.get('processes') is not None:
            processes = int(kwargs['processes'])
        else:
            processes = self.processes

        if kwargs.get('process') is not None:
            process = int(kwargs['process'])
        else:
            process = self.process

        if process < 0:
            raise ValueError(
                'process must be an integer greater than or equal to 0')

        if processes < 0:
            raise ValueError(
                'processes must be an integer greater than or equal to 0')

        if processes and process >= processes:
            raise ValueError(
                'process must be less than or equal to processes')

        return processes, process

    def delete_object(self, actual_obj, timestamp, container, obj):
        start_time = time()
        try:
            self.delete_actual_object(actual_obj, timestamp)
            self.swift.delete_object(self.expiring_objects_account,
                                     container, obj)
            self.report_objects += 1
            self.logger.increment('objects')
        except (Exception, Timeout) as err:
            self.logger.increment('errors')
            self.logger.exception(
                _('Exception while deleting object %s %s %s') %
                (container, obj, str(err)))
        self.logger.timing_since('timing', start_time)
        self.report()

    def delete_actual_object(self, actual_obj, timestamp):
        """
        Deletes the end-user object indicated by the actual object name given
        '<account>/<container>/<object>' if and only if the X-Delete-At value
        of the object is exactly the timestamp given.

        :param actual_obj: The name of the end-user object to delete:
                           '<account>/<container>/<object>'
        :param timestamp: The timestamp the X-Delete-At value must match to
                          perform the actual delete.
        """
        path = '/v1/' + urllib.quote(actual_obj.lstrip('/'))
        self.swift.make_request('DELETE', path,
                                {'X-If-Delete-At': str(timestamp)},
                                (2, HTTP_NOT_FOUND, HTTP_PRECONDITION_FAILED))

# 需要导入模块: from swift.common.internal_client import InternalClient [as 别名]
# 或者: from swift.common.internal_client.InternalClient import make_request [as 别名]

#.........这里部分代码省略.........
            ts = int(float(ts))

            for o in self.swift.iter_objects(self.sample_account, container):
                name = o['name']
                objs_to_delete.append(name)
                ts, bytes_rv, bytes_st, trans_id, client_ip = name.split('/')
                bill_type = self.get_billtype_by_client_ip(client_ip, ts)
                bytes_recvs[bill_type] = bytes_recvs.get(bill_type,
                                                         0) + int(bytes_rv)
                bytes_sents[bill_type] = bytes_sents.get(bill_type,
                                                         0) + int(bytes_st)
                self.report_objects += 1

            for o in objs_to_delete:
                self.swift.delete_object(self.sample_account, container, o)

            for bill_type, bt_rv in bytes_recvs.items():
                t_object = 'transfer/%d/%d/%d_%d_%d' % (ts, bill_type, bt_rv,
                                                        bytes_sents[bill_type],
                                                        self.report_objects)
                self._hidden_update(tenant_id, t_object)
        except (Exception, Timeout) as err:
            self.logger.increment('errors')
            self.logger.exception(
                _('Exception while aggregating sample %s %s') %
                (container, str(err)))

        self.logger.timing_since('timing', start_time)
        self.report()

    def account_info(self, tenant_id, timestamp):
        path = '/v1/%s/%s?prefix=usage/%d&limit=1' % (self.aggregate_account,
                                                      tenant_id, timestamp)
        resp = self.swift.make_request('GET', path, {}, (2,))
        if len(resp.body) == 0:
            return 0, 0, 0
        usages = resp.body.split('/', 2)[2].rstrip()
        cont_cnt, obj_cnt, bt_used = usages.split('_')
        return int(cont_cnt), int(obj_cnt), int(bt_used)

    def _hidden_update(self, container, obj, method='PUT'):
        hidden_path = '/%s/%s/%s' % (self.aggregate_account, container, obj)
        part, nodes = self.container_ring.get_nodes(self.aggregate_account,
                                                    container)
        for node in nodes:
            ip = node['ip']
            port = node['port']
            dev = node['device']
            action_headers = dict()
            action_headers['user-agent'] = 'aggregator'
            action_headers['X-Timestamp'] = normalize_timestamp(time())
            action_headers['referer'] = 'aggregator-daemon'
            action_headers['x-size'] = '0'
            action_headers['x-content-type'] = "text/plain"
            action_headers['x-etag'] = 'd41d8cd98f00b204e9800998ecf8427e'

            conn = http_connect(ip, port, dev, part, method, hidden_path,
                                action_headers)
            response = conn.getresponse()
            response.read()

    def fillup_lossed_usage_data(self, tenants):
        now = (float(time()) // self.sample_rate) * self.sample_rate
        path = '/v1/%s/%s?prefix=usage/%d&limit=1'

        for t in tenants:

# 需要导入模块: from swift.common.internal_client import InternalClient [as 别名]
# 或者: from swift.common.internal_client.InternalClient import make_request [as 别名]

#.........这里部分代码省略.........

            self.swift.delete_object(self.restoring_object_account,
                                     self.todo_container, actual_obj)
            self.report_objects += 1
            self.logger.increment('start')
        except (Exception, Timeout) as err:
            self.logger.increment('errors')
            report_exception(self.logger.exception,
                             _('Exception while restoring object %s. %s') %
                             (obj, str(err)), self.client)
        self.logger.timing_since('timing', start_time)
        self.report()

    def get_archiveid(self, account, container, obj):
        glacier_account = '%s%s' % (self.glacier_account_prefix, account)

        glacier_obj = None
        for o in get_objects_by_prefix(glacier_account, container, obj,
                                       swift_client=self.swift):
            name = get_glacier_objname_from_hidden_object(o)
            if name == obj:
                glacier_obj = o
                break
        if glacier_obj is None:
            return None

        return get_glacier_key_from_hidden_object(glacier_obj)

    def check_object_restored(self, restoring_object):
        actual_obj = get_glacier_objname_from_hidden_object(restoring_object)
        jobId = get_glacier_key_from_hidden_object(restoring_object)
        try:
            path = '/v1/%s' % actual_obj
            resp = self.swift.make_request('GET', path, {}, (2, 4,))
            if resp.status_int == 404:
                raise Exception('Object Not Found: %s' % actual_obj)

            job = self.glacier.get_job(job_id=jobId)
            if not job.completed:
                return
            self.complete_restore(actual_obj, job)
        except Exception as e:
            # Job ID가 만료될 경우 다시 restore 를 시도한다.
            if not e.message.startswith('Object Not Found:'):
                self.start_object_restoring(actual_obj)
            self.logger.info(e)

        self.swift.delete_object(self.restoring_object_account,
                                 self.restoring_container, restoring_object)

    def complete_restore(self, actual_obj, job):
        tmppath = tempfile.NamedTemporaryFile(bufsize=0, delete=False,
                                              dir=self.glacier_tmpdir).name
        try:
            job.download_to_file(filename=tmppath)

            prefix = 'X-Object-Meta'
            a, c, o = actual_obj.split('/', 2)
            metadata = self.swift.get_object_metadata(a, c, o,
                                                      metadata_prefix=prefix)
            metadata = {'X-Object-Meta' + key: value for key, value in metadata
            .iteritems()}
            days = int(metadata['X-Object-Meta-s3-restore-expire-days'])
            exp_time = normalize_delete_at_timestamp(calc_nextDay(time()) +
                                                     (days - 1) * 86400)

# 需要导入模块: from swift.common.internal_client import InternalClient [as 别名]
# 或者: from swift.common.internal_client.InternalClient import make_request [as 别名]

#.........这里部分代码省略.........
            elapsed = time() - begin
            if elapsed < self.interval:
                sleep(random() * (self.interval - elapsed))

    def get_process_values(self, kwargs):
        """
        Gets the processes, process from the kwargs if those values exist.

        Otherwise, return processes, process set in the config file.

        :param kwargs: Keyword args passed into the run_forever(), run_once()
                       methods.  They have values specified on the command
                       line when the daemon is run.
        """
        if kwargs.get('processes') is not None:
            processes = int(kwargs['processes'])
        else:
            processes = self.processes

        if kwargs.get('process') is not None:
            process = int(kwargs['process'])
        else:
            process = self.process

        if process < 0:
            raise ValueError(
                'process must be an integer greater than or equal to 0')

        if processes < 0:
            raise ValueError(
                'processes must be an integer greater than or equal to 0')

        if processes and process >= processes:
            raise ValueError(
                'process must be less than or equal to processes')

        return processes, process

    def delete_object(self, hidden_container, obj):
        start_time = time()
        try:
            account, container, object = obj.split('/', 2)
            lifecycle = Lifecycle(account, container, object,
                                  swift_client=self.swift)
            object_header = lifecycle.object.headers
            object_rule = lifecycle.get_object_rule_by_action('Expiration')
            last_modified = gmt_to_timestamp(object_header['Last-Modified'])

            validation_flg = lifecycle.object_lifecycle_validation()
            if (validation_flg == LIFECYCLE_OK) or \
                    (validation_flg == DISABLED_TRANSITION):
                times = calc_when_actions_do(object_rule, last_modified)
                actual_expire_time = int(times['Expiration'])
                if actual_expire_time == int(hidden_container):
                    self.delete_actual_object(obj)
                if lifecycle.get_s3_storage_class() == 'GLACIER':
                    self.delete_glacier_object(obj)
                self.report_objects += 1
                self.logger.increment('objects')
                self.swift.delete_object(self.s3_expiring_objects_account,
                                         hidden_container, obj)
        except (Exception, Timeout) as err:
            self.logger.increment('errors')
            report_exception(self.logger,
                             _('Exception while deleting object %s %s %s') %
                             (hidden_container, obj, str(err)), self.client)

        self.logger.timing_since('timing', start_time)
        self.report()

    def delete_glacier_object(self, obj):
        account, container, prefix = obj.split('/', 2)
        glacier_hidden_account = self.glacier_account_prefix + account
        objs = get_objects_by_prefix(glacier_hidden_account, container, prefix,
                                     swift_client=self.swift)

        glacier_obj = None
        for o in objs:
            name = get_glacier_objname_from_hidden_object(o)
            if name == prefix:
                glacier_obj = o
                break

        glacier_archive_id = get_glacier_key_from_hidden_object(glacier_obj)
        self.glacier.delete_archive(glacier_archive_id)
        self.swift.delete_object(glacier_hidden_account, container,
                                 glacier_obj)

    def delete_actual_object(self, obj):
        """
        Deletes the end-user object indicated by the actual object name given
        '<account>/<container>/<object>' if and only if the X-Delete-At value
        of the object is exactly the timestamp given.

        :param obj: The name of the end-user object to delete:
                           '<account>/<container>/<object>'
        """
        path = '/v1/' + urllib.quote(obj.lstrip('/'))
        self.swift.make_request('DELETE', path,
                                {}, (2, HTTP_NOT_FOUND))

# 需要导入模块: from swift.common.internal_client import InternalClient [as 别名]
# 或者: from swift.common.internal_client.InternalClient import make_request [as 别名]

#.........这里部分代码省略.........
        except (Exception, Timeout):
            report_exception(self.logger, _('Unhandled exception'), self.client)

    def run_forever(self, *args, **kwargs):
        """
        Executes passes forever, looking for objects to expire.

        :param args: Extra args to fulfill the Daemon interface; this daemon
                     has no additional args.
        :param kwargs: Extra keyword args to fulfill the Daemon interface; this
                       daemon has no additional keyword args.
        """
        sleep(random() * self.interval)
        while True:
            begin = time()
            try:
                self.run_once(*args, **kwargs)
            except (Exception, Timeout):
                report_exception(self.logger, _('Unhandled exception'), self.client)
            elapsed = time() - begin
            if elapsed < self.interval:
                sleep(random() * (self.interval - elapsed))

    def get_process_values(self, kwargs):
        """
        Gets the processes, process from the kwargs if those values exist.

        Otherwise, return processes, process set in the config file.

        :param kwargs: Keyword args passed into the run_forever(), run_once()
                       methods.  They have values specified on the command
                       line when the daemon is run.
        """
        if kwargs.get('processes') is not None:
            processes = int(kwargs['processes'])
        else:
            processes = self.processes

        if kwargs.get('process') is not None:
            process = int(kwargs['process'])
        else:
            process = self.process

        if process < 0:
            raise ValueError(
                'process must be an integer greater than or equal to 0')

        if processes < 0:
            raise ValueError(
                'processes must be an integer greater than or equal to 0')

        if processes and process >= processes:
            raise ValueError(
                'process must be less than or equal to processes')

        return processes, process

    def transition_object(self, container, obj):
        start_time = time()
        try:
            obj_account, obj_container, obj_object = obj.split('/', 2)

            lifecycle = Lifecycle(obj_account, obj_container, obj_object,
                                  swift_client=self.swift)

            if is_success(lifecycle.object.status):
                object_header = lifecycle.object.headers
                object_rule = lifecycle.get_object_rule_by_action(
                    'Transition')
                last_modified = object_header['Last-Modified']
                last_modified = gmt_to_timestamp(last_modified)

                validation_flg = lifecycle.object_lifecycle_validation()
                if (validation_flg == LIFECYCLE_OK) or \
                        (validation_flg == DISABLED_EXPIRATION):
                    times = calc_when_actions_do(object_rule, last_modified)
                    actual_expire_time = int(times['Transition'])
                    if actual_expire_time == int(container):
                        self.request_transition(obj)

                    self.swift.delete_object(self.s3_tr_objects_account,
                                             container, obj)
        except (Exception, Timeout) as err:
            self.logger.increment('errors')
            report_exception(self.logger,
                             _('Exception while transitioning object %s %s %s') %
                             (container, obj, str(err)), self.client)
        self.logger.timing_since('timing', start_time)
        self.report()

    def request_transition(self, actual_obj):
        path = '/v1/' + urllib.quote(actual_obj.lstrip('/'))
        headers = {GLACIER_FLAG_META: True,
                   'X-S3-Object-Transition': True}
        resp = self.swift.make_request('POST', path, headers, (2, 5))

        if resp.status_int == 500:
            raise Exception(resp.body)
        self.report_objects += 1
        self.logger.increment('objects')