Python header_key_dict.HeaderKeyDict类代码示例

    def _clean_outgoing_headers(self, headers):
        """
        Removes any headers as per the middleware configuration for
        outgoing responses.

        :param headers: A WSGI start_response style list of headers,
                        [('header1', 'value), ('header2', 'value),
                         ...]
        :returns: The same headers list, but with some headers
                  removed as per the middlware configuration for
                  outgoing responses.
        """
        headers = HeaderKeyDict(headers)
        for h in headers.keys():
            if h in self.outgoing_allow_headers:
                continue
            for p in self.outgoing_allow_headers_startswith:
                if h.startswith(p):
                    break
            else:
                if h in self.outgoing_remove_headers:
                    del headers[h]
                    continue
                for p in self.outgoing_remove_headers_startswith:
                    if h.startswith(p):
                        del headers[h]
                        break
        return headers.items()

    def test_get_undersize_segment(self):
        # If we send a Content-Length header to the client, it's based on the
        # container listing. If a segment gets smaller by the time we get to
        # it (like if a client uploads a smaller segment w/the same name), we
        # need to raise an exception so that the connection will be closed by
        # the WSGI server. Otherwise, the WSGI server will be waiting for the
        # next request, the client will still be waiting for the rest of the
        # response, and nobody will be happy.

        # Shrink it by a single byte
        self.app.register(
            'GET', '/v1/AUTH_test/c/seg_03',
            swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")},
            'cccc')

        req = swob.Request.blank(
            '/v1/AUTH_test/mancon/manifest',
            environ={'REQUEST_METHOD': 'GET'})
        status, headers, body, exc = self.call_dlo(req, expect_exception=True)
        headers = HeaderKeyDict(headers)

        self.assertEqual(status, '200 OK')  # sanity check
        self.assertEqual(headers.get('Content-Length'), '25')  # sanity check
        self.assertEqual(body, 'aaaaabbbbbccccdddddeeeee')
        self.assertTrue(isinstance(exc, exceptions.SegmentError))

    def test_get_oversize_segment(self):
        # If we send a Content-Length header to the client, it's based on the
        # container listing. If a segment gets bigger by the time we get to it
        # (like if a client uploads a bigger segment w/the same name), we need
        # to not send anything beyond the length we promised. Also, we should
        # probably raise an exception.

        # This is now longer than the original seg_03+seg_04+seg_05 combined
        self.app.register(
            'GET', '/v1/AUTH_test/c/seg_03',
            swob.HTTPOk, {'Content-Length': '20', 'Etag': 'seg03-etag'},
            'cccccccccccccccccccc')

        req = swob.Request.blank(
            '/v1/AUTH_test/mancon/manifest',
            environ={'REQUEST_METHOD': 'GET'})
        status, headers, body = self.call_dlo(req)
        headers = HeaderKeyDict(headers)

        self.assertEqual(status, '200 OK')  # sanity check
        self.assertEqual(headers.get('Content-Length'), '25')  # sanity check
        self.assertEqual(body, b'aaaaabbbbbccccccccccccccc')
        self.assertEqual(
            self.app.calls,
            [('GET', '/v1/AUTH_test/mancon/manifest'),
             ('GET', '/v1/AUTH_test/c?prefix=seg'),
             ('GET', '/v1/AUTH_test/c/seg_01?multipart-manifest=get'),
             ('GET', '/v1/AUTH_test/c/seg_02?multipart-manifest=get'),
             ('GET', '/v1/AUTH_test/c/seg_03?multipart-manifest=get')])

 def test_keys(self):
     headers = HeaderKeyDict()
     headers['content-length'] = 20
     headers['cOnTent-tYpe'] = 'text/plain'
     headers['SomeThing-eLse'] = 'somevalue'
     self.assertEqual(
         set(headers.keys()),
         set(('Content-Length', 'Content-Type', 'Something-Else')))

 def test_pop(self):
     headers = HeaderKeyDict()
     headers['content-length'] = 20
     headers['cOntent-tYpe'] = 'text/plain'
     self.assertEqual(headers.pop('content-Length'), '20')
     self.assertEqual(headers.pop('Content-type'), 'text/plain')
     self.assertEqual(headers.pop('Something-Else', 'somevalue'),
                      'somevalue')

    def process_object_update(self, update_path, device, policy):
        """
        Process the object information to be updated and update.

        :param update_path: path to pickled object update file
        :param device: path to device
        :param policy: storage policy of object update
        """
        try:
            update = pickle.load(open(update_path, 'rb'))
        except Exception:
            self.logger.exception(
                _('ERROR Pickle problem, quarantining %s'), update_path)
            self.stats.quarantines += 1
            self.logger.increment('quarantines')
            target_path = os.path.join(device, 'quarantined', 'objects',
                                       os.path.basename(update_path))
            renamer(update_path, target_path, fsync=False)
            return
        successes = update.get('successes', [])
        part, nodes = self.get_container_ring().get_nodes(
            update['account'], update['container'])
        obj = '/%s/%s/%s' % \
              (update['account'], update['container'], update['obj'])
        headers_out = HeaderKeyDict(update['headers'])
        headers_out['user-agent'] = 'object-updater %s' % os.getpid()
        headers_out.setdefault('X-Backend-Storage-Policy-Index',
                               str(int(policy)))
        events = [spawn(self.object_update,
                        node, part, update['op'], obj, headers_out)
                  for node in nodes if node['id'] not in successes]
        success = True
        new_successes = False
        for event in events:
            event_success, node_id = event.wait()
            if event_success is True:
                successes.append(node_id)
                new_successes = True
            else:
                success = False
        if success:
            self.stats.successes += 1
            self.logger.increment('successes')
            self.logger.debug('Update sent for %(obj)s %(path)s',
                              {'obj': obj, 'path': update_path})
            self.stats.unlinks += 1
            self.logger.increment('unlinks')
            os.unlink(update_path)
        else:
            self.stats.failures += 1
            self.logger.increment('failures')
            self.logger.debug('Update failed for %(obj)s %(path)s',
                              {'obj': obj, 'path': update_path})
            if new_successes:
                update['successes'] = successes
                write_pickle(update, update_path, os.path.join(
                    device, get_tmp_dir(policy)))

    def test_head_large_object_too_many_segments(self):
        req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
                                 environ={'REQUEST_METHOD': 'HEAD'})
        with mock.patch(LIMIT, 3):
            status, headers, body = self.call_dlo(req)
        headers = HeaderKeyDict(headers)

        # etag is manifest's etag
        self.assertEqual(headers["Etag"], "etag-manyseg")
        self.assertIsNone(headers.get("Content-Length"))

 def test_get_suffix_range_many_segments(self):
     req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
                              environ={'REQUEST_METHOD': 'GET'},
                              headers={'Range': 'bytes=-5'})
     with mock.patch(LIMIT, 3):
         status, headers, body = self.call_dlo(req)
     headers = HeaderKeyDict(headers)
     self.assertEqual(status, "200 OK")
     self.assertIsNone(headers.get("Content-Length"))
     self.assertIsNone(headers.get("Content-Range"))
     self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")

 def test_get_range_many_segments_satisfiability_unknown(self):
     req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
                              environ={'REQUEST_METHOD': 'GET'},
                              headers={'Range': 'bytes=10-22'})
     with mock.patch(LIMIT, 3):
         status, headers, body = self.call_dlo(req)
     headers = HeaderKeyDict(headers)
     self.assertEqual(status, "200 OK")
     # this requires multiple pages of container listing, so we can't send
     # a Content-Length header
     self.assertIsNone(headers.get("Content-Length"))
     self.assertEqual(body, b"aaaaabbbbbcccccdddddeeeee")

    def test_get_response_headers_with_legacy_data(self):
        broker = backend.AccountBroker(':memory:', account='a')
        now = time.time()
        with mock.patch('time.time', new=lambda: now):
            broker.initialize(Timestamp(now).internal)
        # add some container data
        ts = (Timestamp(t).internal for t in itertools.count(int(now)))
        total_containers = 0
        total_objects = 0
        total_bytes = 0
        for policy in POLICIES:
            delete_timestamp = next(ts)
            put_timestamp = next(ts)
            object_count = int(policy)
            bytes_used = int(policy) * 10
            broker.put_container('c-%s' % policy.name, put_timestamp,
                                 delete_timestamp, object_count, bytes_used,
                                 int(policy))
            total_containers += 1
            total_objects += object_count
            total_bytes += bytes_used
        expected = HeaderKeyDict({
            'X-Account-Container-Count': total_containers,
            'X-Account-Object-Count': total_objects,
            'X-Account-Bytes-Used': total_bytes,
            'X-Timestamp': Timestamp(now).normal,
            'X-PUT-Timestamp': Timestamp(now).normal,
        })
        for policy in POLICIES:
            prefix = 'X-Account-Storage-Policy-%s-' % policy.name
            expected[prefix + 'Object-Count'] = int(policy)
            expected[prefix + 'Bytes-Used'] = int(policy) * 10
        orig_policy_stats = broker.get_policy_stats

        def stub_policy_stats(*args, **kwargs):
            policy_stats = orig_policy_stats(*args, **kwargs)
            for stats in policy_stats.values():
                # legacy db's won't return container_count
                del stats['container_count']
            return policy_stats
        broker.get_policy_stats = stub_policy_stats
        resp_headers = utils.get_response_headers(broker)
        per_policy_container_headers = [
            h for h in resp_headers if
            h.lower().startswith('x-account-storage-policy-') and
            h.lower().endswith('-container-count')]
        self.assertFalse(per_policy_container_headers)
        for key, value in resp_headers.items():
            expected_value = expected.pop(key)
            self.assertEqual(expected_value, str(value),
                             'value for %r was %r not %r' % (
                                 key, value, expected_value))
        self.assertFalse(expected)

    def handle_get_head_symlink(self, req):
        """
        Handle get/head request when client sent parameter ?symlink=get

        :param req: HTTP GET or HEAD object request with param ?symlink=get
        :returns: Response Iterator
        """
        resp = self._app_call(req.environ)
        response_header_dict = HeaderKeyDict(self._response_headers)
        symlink_sysmeta_to_usermeta(response_header_dict)
        self._response_headers = response_header_dict.items()
        return resp

 def test_get_multi_range(self):
     # DLO doesn't support multi-range GETs. The way that you express that
     # in HTTP is to return a 200 response containing the whole entity.
     req = swob.Request.blank('/v1/AUTH_test/mancon/manifest-many-segments',
                              environ={'REQUEST_METHOD': 'GET'},
                              headers={'Range': 'bytes=5-9,15-19'})
     with mock.patch(LIMIT, 3):
         status, headers, body = self.call_dlo(req)
     headers = HeaderKeyDict(headers)
     self.assertEqual(status, "200 OK")
     self.assertIsNone(headers.get("Content-Length"))
     self.assertIsNone(headers.get("Content-Range"))
     self.assertEqual(body, b'aaaaabbbbbcccccdddddeeeee')

 def test_get_range(self):
     req = swob.Request.blank('/v1/AUTH_test/mancon/manifest',
                              environ={'REQUEST_METHOD': 'GET'},
                              headers={'Range': 'bytes=8-17'})
     status, headers, body = self.call_dlo(req)
     headers = HeaderKeyDict(headers)
     self.assertEqual(status, "206 Partial Content")
     self.assertEqual(headers["Content-Length"], "10")
     self.assertEqual(body, b'bbcccccddd')
     expected_etag = '"%s"' % md5hex(
         md5hex("aaaaa") + md5hex("bbbbb") + md5hex("ccccc") +
         md5hex("ddddd") + md5hex("eeeee"))
     self.assertEqual(headers.get("Etag"), expected_etag)

    def handle(self, req, start_response):
        app_resp = self._app_call(req.environ)

        if is_success(self._get_status_int()):
            # only decrypt body of 2xx responses
            headers = HeaderKeyDict(self._response_headers)
            content_type = headers.get('content-type', '').split(';', 1)[0]
            if content_type == 'application/json':
                app_resp = self.process_json_resp(req, app_resp)

        start_response(self._response_status,
                       self._response_headers,
                       self._response_exc_info)

        return app_resp

    def test_get_undersize_segment_range(self):
        # Shrink it by a single byte
        self.app.register(
            'GET', '/v1/AUTH_test/c/seg_03',
            swob.HTTPOk, {'Content-Length': '4', 'Etag': md5hex("cccc")},
            'cccc')

        req = swob.Request.blank(
            '/v1/AUTH_test/mancon/manifest',
            environ={'REQUEST_METHOD': 'GET'},
            headers={'Range': 'bytes=0-14'})
        status, headers, body = self.call_dlo(req)
        headers = HeaderKeyDict(headers)

        self.assertEqual(status, '206 Partial Content')  # sanity check
        self.assertEqual(headers.get('Content-Length'), '15')  # sanity check
        self.assertEqual(body, b'aaaaabbbbbcccc')