Python InternalClient.get_object方法代码示例

# 需要导入模块: from swift.common.internal_client import InternalClient [as 别名]
# 或者: from swift.common.internal_client.InternalClient import get_object [as 别名]
    def download(self, acc, container, u_agent, delay=0, request_tries=3):
        self.logger.info('Prefetching objects with InternalClient with ' + str(delay) + ' seconds of delay.')
        time.sleep(delay)
        swift = InternalClient(PROXY_PATH, u_agent, request_tries=request_tries)
        headers = {}

        prefetch_list = []
        bytes_count = 0
        for o in swift.iter_objects(acc, container):
            if bytes_count + int(o['bytes']) < self.cache_max_size:
                prefetch_list.append(o['name'])
                bytes_count += int(o['bytes'])
            else:
                break

        for name in prefetch_list:
            object_path = '/v1/' + acc + '/' + container + '/' + name
            oid = hashlib.md5(object_path).hexdigest()

            status, resp_headers, it = swift.get_object(acc, container, name, headers, ACCEPTABLE_STATUS)

            object_size = int(resp_headers.get('Content-Length'))
            object_etag = resp_headers.get('Etag')

            object_storage_policy_id = '0'  # FIXME hardcoded
            to_evict = self.cache.access_cache("PUT", oid, object_size, object_etag, object_storage_policy_id)
            for ev_object_id in to_evict:
                os.remove(os.path.join(self.cache_path, ev_object_id))

            self.logger.info('Prefetch Filter - Object ' + name + ' stored in cache with ID: ' + oid)
            with open(os.path.join(self.cache_path, oid), 'w') as f:
                for el in it:
                    f.write(el)

# 需要导入模块: from swift.common.internal_client import InternalClient [as 别名]
# 或者: from swift.common.internal_client.InternalClient import get_object [as 别名]

#.........这里部分代码省略.........
        #: Time of last stats report.
        self.reported = time()
        self.swift_dir = conf.get('swift_dir', '/etc/swift')
        #: swift.common.ring.Ring for locating containers.
        self.container_ring = container_ring or Ring(self.swift_dir,
                                                     ring_name='container')
        bind_ip = conf.get('bind_ip', '0.0.0.0')
        self._myips = whataremyips(bind_ip)
        self._myport = int(conf.get('bind_port', 6001))
        swift.common.db.DB_PREALLOCATION = \
            config_true_value(conf.get('db_preallocation', 'f'))
        self.conn_timeout = float(conf.get('conn_timeout', 5))
        request_tries = int(conf.get('request_tries') or 3)

        internal_client_conf_path = conf.get('internal_client_conf_path')
        if not internal_client_conf_path:
            self.logger.warning(
                _('Configuration option internal_client_conf_path not '
                  'defined. Using default configuration, See '
                  'internal-client.conf-sample for options'))
            internal_client_conf = ConfigString(ic_conf_body)
        else:
            internal_client_conf = internal_client_conf_path
        try:
            self.swift = InternalClient(
                internal_client_conf, 'Swift Container Sync', request_tries)
        except IOError as err:
            if err.errno != errno.ENOENT:
                raise
            raise SystemExit(
                _('Unable to load internal client from config: %r (%s)') %
                (internal_client_conf_path, err))

    def get_object_ring(self, policy_idx):
        """
        Get the ring object to use based on its policy.

        :policy_idx: policy index as defined in swift.conf
        :returns: appropriate ring object
        """
        return POLICIES.get_object_ring(policy_idx, self.swift_dir)

    def run_forever(self, *args, **kwargs):
        """
        Runs container sync scans until stopped.
        """
        sleep(random() * self.interval)
        while True:
            begin = time()
            for path in self.sync_store.synced_containers_generator():
                self.container_sync(path)
                if time() - self.reported >= 3600:  # once an hour
                    self.report()
            elapsed = time() - begin
            if elapsed < self.interval:
                sleep(self.interval - elapsed)

    def run_once(self, *args, **kwargs):
        """
        Runs a single container sync scan.
        """
        self.logger.info(_('Begin container sync "once" mode'))
        begin = time()
        for path in self.sync_store.synced_containers_generator():
            self.container_sync(path)
            if time() - self.reported >= 3600:  # once an hour

# 需要导入模块: from swift.common.internal_client import InternalClient [as 别名]
# 或者: from swift.common.internal_client.InternalClient import get_object [as 别名]

#.........这里部分代码省略.........
        Perform object reconciliation.

        :param account: the account name of the misplaced object
        :param container: the container name of the misplaced object
        :param obj: the object name
        :param q_policy_index: the policy index of the source indicated by the
                               queue entry.
        :param q_ts: the timestamp of the misplaced object
        :param q_op: the operation of the misplaced request
        :param path: the full path of the misplaced object for logging

        :returns: True to indicate the request is fully processed
                  successfully, otherwise False.
        """
        container_policy_index = direct_get_container_policy_index(
            self.swift.container_ring, account, container)
        if container_policy_index is None:
            self.stats_log('unavailable_container', '%r (%f) unable to '
                           'determine the destination policy_index',
                           path, q_ts)
            return False
        if container_policy_index == q_policy_index:
            self.stats_log('noop_object', '%r (%f) container policy_index '
                           '%s matches queue policy index %s', path, q_ts,
                           container_policy_index, q_policy_index)
            return True

        # check if object exists in the destination already
        self.logger.debug('checking for %r (%f) in destination '
                          'policy_index %s', path, q_ts,
                          container_policy_index)
        headers = {
            'X-Backend-Storage-Policy-Index': container_policy_index}
        dest_obj = self.swift.get_object_metadata(account, container, obj,
                                                  headers=headers,
                                                  acceptable_statuses=(2, 4))
        dest_ts = Timestamp(dest_obj.get('x-backend-timestamp', 0))
        if dest_ts >= q_ts:
            self.stats_log('found_object', '%r (%f) in policy_index %s '
                           'is newer than queue (%f)', path, dest_ts,
                           container_policy_index, q_ts)
            return self.throw_tombstones(account, container, obj, q_ts,
                                         q_policy_index, path)

        # object is misplaced
        self.stats_log('misplaced_object', '%r (%f) in policy_index %s '
                       'should be in policy_index %s', path, q_ts,
                       q_policy_index, container_policy_index)

        # fetch object from the source location
        self.logger.debug('fetching %r (%f) from storage policy %s', path,
                          q_ts, q_policy_index)
        headers = {
            'X-Backend-Storage-Policy-Index': q_policy_index}
        try:
            source_obj_status, source_obj_info, source_obj_iter = \
                self.swift.get_object(account, container, obj,
                                      headers=headers,
                                      acceptable_statuses=(2, 4))
        except UnexpectedResponse as err:
            source_obj_status = err.resp.status_int
            source_obj_info = {}
            source_obj_iter = None

        source_ts = Timestamp(source_obj_info.get('x-backend-timestamp', 0))
        if source_obj_status == 404 and q_op == 'DELETE':