Python swob.HeaderKeyDict类代码示例

    def generate_request_headers(self, orig_req=None, additional=None,
                                 transfer=False):
        """
        Create a list of headers to be used in backend requets

        :param orig_req: the original request sent by the client to the proxy
        :param additional: additional headers to send to the backend
        :param transfer: If True, transfer headers from original client request
        :returns: a dictionary of headers
        """
        # Use the additional headers first so they don't overwrite the headers
        # we require.
        headers = HeaderKeyDict(additional) if additional else HeaderKeyDict()
        if transfer:
            self.transfer_headers(orig_req.headers, headers)
        headers.setdefault('x-timestamp', normalize_timestamp(time.time()))
        if orig_req:
            referer = orig_req.as_referer()
        else:
            referer = ''
        headers['x-trans-id'] = self.trans_id
        headers['connection'] = 'close'
        headers['user-agent'] = 'proxy-server %s' % os.getpid()
        headers['referer'] = referer
        return headers

    def _clean_outgoing_headers(self, headers):
        """
        Removes any headers as per the middleware configuration for
        outgoing responses.

        :param headers: A WSGI start_response style list of headers,
                        [('header1', 'value), ('header2', 'value),
                         ...]
        :returns: The same headers list, but with some headers
                  removed as per the middlware configuration for
                  outgoing responses.
        """
        headers = HeaderKeyDict(headers)
        for h in headers.keys():
            remove = h in self.outgoing_remove_headers
            if not remove:
                for p in self.outgoing_remove_headers_startswith:
                    if h.startswith(p):
                        remove = True
                        break
            if remove:
                if h in self.outgoing_allow_headers:
                    remove = False
            if remove:
                for p in self.outgoing_allow_headers_startswith:
                    if h.startswith(p):
                        remove = False
                        break
            if remove:
                del headers[h]
        return headers.items()

    def generate_request_headers(self, orig_req=None, additional=None, transfer=False):
        """
        Create a list of headers to be used in backend requets

        :param orig_req: the original request sent by the client to the proxy
        :param additional: additional headers to send to the backend
        :param transfer: If True, transfer headers from original client request
        :returns: a dictionary of headers
        """
        # Use the additional headers first so they don't overwrite the headers
        # we require.
        headers = HeaderKeyDict(additional) if additional else HeaderKeyDict()
        if transfer:
            self.transfer_headers(orig_req.headers, headers)
        if "x-timestamp" not in headers:
            headers["x-timestamp"] = normalize_timestamp(time.time())
        if orig_req:
            referer = orig_req.as_referer()
        else:
            referer = ""
        headers.update(
            {
                "x-trans-id": self.trans_id,
                "connection": "close",
                "user-agent": "proxy-server %s" % os.getpid(),
                "referer": referer,
            }
        )
        return headers

    def publish_search(self, path, req):
        """ Publish a verify request on the queue to gate engine """

        headers = HeaderKeyDict(req.headers)
        #self.logger.debug("SWIFTSEARCH avaiable headers: %s" % (headers.items()))

        # TODO(mlopezc1) is this actually faster than a regular regex with match?
        #   swift code loves this pattern (ex tempurl middleware) but not sure how
        #   will this actually perform in high use. Perf comparison later?
        for k, v in headers.items():
            # if header not in the whitelist of allowed full header names or *
            if k not in self.index_headers:
                #self.logger.debug("SWIFTSEARCH k=%s not in %s" % (k, self.index_headers))
                for h in self.index_headers_startwith:
                    if not k.startswith(h):
                        #self.logger.debug("SWIFTSEARCH k=%s not allowed" % (k))
                        del headers[k]

        self.logger.debug("SWIFTSEARCH sending metadata for indexing: %s" % (headers.items()))

        # TODO(mlopez1) what about renaming keys to something more human ? the X- and title format is kinda weird
        exchange = kombu.Exchange(self.exc_str, self.exc_type, durable=self.exc_durable)
        queue = kombu.Queue('search', exchange=exchange, routing_key='search')

        with kombu.Connection(self.conn_str) as connection:
            with connection.Producer(serializer='json') as producer:
                producer.publish({'id': b64encode(path), 'path': path, 'metadata': headers},
                                 exchange=exchange, routing_key='search', declare=[queue])

        return True

    def test_206_multiple_ranges(self):
        fr = FakeResponse(
            206,
            {'Content-Type': 'multipart/byteranges; boundary=asdfasdfasdf'},
            ("--asdfasdfasdf\r\n"
             "Content-Type: application/lunch\r\n"
             "Content-Range: bytes 0-3/10\r\n"
             "\r\n"
             "sand\r\n"
             "--asdfasdfasdf\r\n"
             "Content-Type: application/lunch\r\n"
             "Content-Range: bytes 6-9/10\r\n"
             "\r\n"
             "ches\r\n"
             "--asdfasdfasdf--"))

        doc_iters = http_response_to_document_iters(fr)

        first_byte, last_byte, length, headers, body = next(doc_iters)
        self.assertEqual(first_byte, 0)
        self.assertEqual(last_byte, 3)
        self.assertEqual(length, 10)
        header_dict = HeaderKeyDict(headers)
        self.assertEqual(header_dict.get('Content-Type'), 'application/lunch')
        self.assertEqual(body.read(), 'sand')

        first_byte, last_byte, length, headers, body = next(doc_iters)
        self.assertEqual(first_byte, 6)
        self.assertEqual(last_byte, 9)
        self.assertEqual(length, 10)
        header_dict = HeaderKeyDict(headers)
        self.assertEqual(header_dict.get('Content-Type'), 'application/lunch')
        self.assertEqual(body.read(), 'ches')

        self.assertRaises(StopIteration, next, doc_iters)

 def cookie_resp(status, response_headers, exc_info=None):
     resp_headers = HeaderKeyDict(response_headers)
     if 'x-auth-token' in resp_headers:
         auth_token = resp_headers['x-auth-token']
         expires_in = int(resp_headers.get('x-auth-token-expires', 0))
         storage_url = resp_headers.get('x-storage-url', '')
         path_parts = urlparse(storage_url)
         domain = path_parts.hostname
         secure = False
         if path_parts.scheme == 'https':
             secure = True
         if auth_token and domain:
             new_cookie = create_auth_cookie('session',
                                             domain,
                                             token=auth_token,
                                             expires_in=expires_in,
                                             secure=secure,
                                             httponly=True)
             response_headers.append(('Set-Cookie', new_cookie))
             new_cookie = create_auth_cookie('storage',
                                             domain,
                                             token=quote(storage_url,
                                                         safe=''),
                                             expires_in=expires_in,
                                             secure=secure)
             response_headers.append(('Set-Cookie', new_cookie))
     return start_response(status, response_headers, exc_info)

    def test_clean_outgoing_headers(self):
        orh = ''
        oah = ''
        hdrs = {'test-header': 'value'}
        hdrs = HeaderKeyDict(tempurl.TempURL(
            None,
            {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
        )._clean_outgoing_headers(hdrs.items()))
        self.assertTrue('test-header' in hdrs)

        orh = 'test-header'
        oah = ''
        hdrs = {'test-header': 'value'}
        hdrs = HeaderKeyDict(tempurl.TempURL(
            None,
            {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
        )._clean_outgoing_headers(hdrs.items()))
        self.assertTrue('test-header' not in hdrs)

        orh = 'test-header-*'
        oah = ''
        hdrs = {'test-header-one': 'value',
                'test-header-two': 'value'}
        hdrs = HeaderKeyDict(tempurl.TempURL(
            None,
            {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
        )._clean_outgoing_headers(hdrs.items()))
        self.assertTrue('test-header-one' not in hdrs)
        self.assertTrue('test-header-two' not in hdrs)

        orh = 'test-header-*'
        oah = 'test-header-two'
        hdrs = {'test-header-one': 'value',
                'test-header-two': 'value'}
        hdrs = HeaderKeyDict(tempurl.TempURL(
            None,
            {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
        )._clean_outgoing_headers(hdrs.items()))
        self.assertTrue('test-header-one' not in hdrs)
        self.assertTrue('test-header-two' in hdrs)

        orh = 'test-header-* test-other-header'
        oah = 'test-header-two test-header-yes-*'
        hdrs = {'test-header-one': 'value',
                'test-header-two': 'value',
                'test-other-header': 'value',
                'test-header-yes': 'value',
                'test-header-yes-this': 'value'}
        hdrs = HeaderKeyDict(tempurl.TempURL(
            None,
            {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
        )._clean_outgoing_headers(hdrs.items()))
        self.assertTrue('test-header-one' not in hdrs)
        self.assertTrue('test-header-two' in hdrs)
        self.assertTrue('test-other-header' not in hdrs)
        self.assertTrue('test-header-yes' not in hdrs)
        self.assertTrue('test-header-yes-this' in hdrs)

    def test_get_response_headers_with_legacy_data(self):
        broker = backend.AccountBroker(':memory:', account='a')
        now = time.time()
        with mock.patch('time.time', new=lambda: now):
            broker.initialize(Timestamp(now).internal)
        # add some container data
        ts = (Timestamp(t).internal for t in itertools.count(int(now)))
        total_containers = 0
        total_objects = 0
        total_bytes = 0
        for policy in POLICIES:
            delete_timestamp = ts.next()
            put_timestamp = ts.next()
            object_count = int(policy)
            bytes_used = int(policy) * 10
            broker.put_container('c-%s' % policy.name, put_timestamp,
                                 delete_timestamp, object_count, bytes_used,
                                 int(policy))
            total_containers += 1
            total_objects += object_count
            total_bytes += bytes_used
        expected = HeaderKeyDict({
            'X-Account-Container-Count': total_containers,
            'X-Account-Object-Count': total_objects,
            'X-Account-Bytes-Used': total_bytes,
            'X-Timestamp': Timestamp(now).normal,
            'X-PUT-Timestamp': Timestamp(now).normal,
        })
        for policy in POLICIES:
            prefix = 'X-Account-Storage-Policy-%s-' % policy.name
            expected[prefix + 'Object-Count'] = int(policy)
            expected[prefix + 'Bytes-Used'] = int(policy) * 10
        orig_policy_stats = broker.get_policy_stats

        def stub_policy_stats(*args, **kwargs):
            policy_stats = orig_policy_stats(*args, **kwargs)
            for stats in policy_stats.values():
                # legacy db's won't return container_count
                del stats['container_count']
            return policy_stats
        broker.get_policy_stats = stub_policy_stats
        resp_headers = utils.get_response_headers(broker)
        per_policy_container_headers = [
            h for h in resp_headers if
            h.lower().startswith('x-account-storage-policy-') and
            h.lower().endswith('-container-count')]
        self.assertFalse(per_policy_container_headers)
        for key, value in resp_headers.items():
            expected_value = expected.pop(key)
            self.assertEqual(expected_value, str(value),
                             'value for %r was %r not %r' % (
                                 key, value, expected_value))
        self.assertFalse(expected)

    def test_fake_swift_sysmeta(self):
        swift = FakeSwift()
        orig_headers = HeaderKeyDict()
        orig_headers.update({sysmeta_header('container', 'acl'): 'test',
                             'x-container-meta-foo': 'bar'})

        swift.register(self.method, self.path, MagicMock(), orig_headers, None)

        self._check_headers(swift, self.method, self.path, orig_headers)

        new_headers = orig_headers.copy()
        del new_headers[sysmeta_header('container', 'acl').title()]
        swift.register(self.method, self.path, MagicMock(), new_headers, None)

        self._check_headers(swift, self.method, self.path, orig_headers)

class FakeConn(object):

    def __init__(self, status, headers=None, body='', **kwargs):
        self.status = status
        try:
            self.reason = RESPONSE_REASONS[self.status][0]
        except Exception:
            self.reason = 'Fake'
        self.body = body
        self.resp_headers = HeaderKeyDict()
        if headers:
            self.resp_headers.update(headers)
        self.with_exc = False
        self.etag = None

    def _update_raw_call_args(self, *args, **kwargs):
        capture_attrs = ('host', 'port', 'method', 'path', 'req_headers',
                         'query_string')
        for attr, value in zip(capture_attrs, args[:len(capture_attrs)]):
            setattr(self, attr, value)
        return self

    def getresponse(self):
        if self.etag:
            self.resp_headers['etag'] = str(self.etag.hexdigest())
        if self.with_exc:
            raise Exception('test')
        return self

    def getheader(self, header, default=None):
        return self.resp_headers.get(header, default)

    def getheaders(self):
        return self.resp_headers.items()

    def read(self, amt=None):
        if amt is None:
            return self.body
        elif isinstance(self.body, six.StringIO):
            return self.body.read(amt)
        else:
            return Exception('Not a StringIO entry')

    def send(self, data):
        if not self.etag:
            self.etag = md5()
        self.etag.update(data)

 def generate_request_headers(self, orig_req=None, additional=None,
                              transfer=False):
     # Use the additional headers first so they don't overwrite the headers
     # we require.
     headers = HeaderKeyDict(additional) if additional else HeaderKeyDict()
     if transfer:
         self.transfer_headers(orig_req.headers, headers)
     if 'x-timestamp' not in headers:
         headers['x-timestamp'] = normalize_timestamp(time.time())
     if orig_req:
         referer = orig_req.as_referer()
     else:
         referer = ''
     headers.update({'x-trans-id': self.trans_id,
                     'connection': 'close',
                     'user-agent': 'proxy-server %s' % os.getpid(),
                     'referer': referer})
     return headers

 def handle_request(self, env, start_response):
     account_id = env.get('REMOTE_USER', None)
     resp = self._app_call(env)
     headers = HeaderKeyDict(self._response_headers)
     if 'x-nexe-cdr-line' in headers and account_id:
         try:
             total_time, line = headers['x-nexe-cdr-line'].split(', ', 1)
             node_lines = re.split(r'\s*,\s*', line)
             total = []
             for rtime, line in zip(*[iter(node_lines)]*2):
                 accounting_info = line.split(' ')
                 total = self.liteacc.cache_accounting_info(account_id, rtime, accounting_info)
                 self.liteacc.queue.put(account_id)
             headers['x-nexe-cdr-total'] = ' '.join([str(t) for t in total])
             self._response_headers = [(k, v) for k, v in headers.iteritems()]
         except ValueError:
             self.logger.warning('Accounting cannot parse CDR entry: %s' % headers['x-nexe-cdr-line'])
     start_response(self._response_status, self._response_headers,
                    self._response_exc_info)
     return resp

 def __init__(self, status, headers=None, body='', **kwargs):
     self.status = status
     try:
         self.reason = RESPONSE_REASONS[self.status][0]
     except Exception:
         self.reason = 'Fake'
     self.body = body
     self.resp_headers = HeaderKeyDict()
     if headers:
         self.resp_headers.update(headers)
     self.with_exc = False
     self.etag = None

    def test_200(self):
        fr = FakeResponse(
            200,
            {'Content-Length': '10', 'Content-Type': 'application/lunch'},
            'sandwiches')

        doc_iters = http_response_to_document_iters(fr)
        first_byte, last_byte, length, headers, body = next(doc_iters)
        self.assertEqual(first_byte, 0)
        self.assertEqual(last_byte, 9)
        self.assertEqual(length, 10)
        header_dict = HeaderKeyDict(headers)
        self.assertEqual(header_dict.get('Content-Length'), '10')
        self.assertEqual(header_dict.get('Content-Type'), 'application/lunch')
        self.assertEqual(body.read(), 'sandwiches')

        self.assertRaises(StopIteration, next, doc_iters)

        fr = FakeResponse(
            200,
            {'Transfer-Encoding': 'chunked',
             'Content-Type': 'application/lunch'},
            'sandwiches')

        doc_iters = http_response_to_document_iters(fr)
        first_byte, last_byte, length, headers, body = next(doc_iters)
        self.assertEqual(first_byte, 0)
        self.assertIsNone(last_byte)
        self.assertIsNone(length)
        header_dict = HeaderKeyDict(headers)
        self.assertEqual(header_dict.get('Transfer-Encoding'), 'chunked')
        self.assertEqual(header_dict.get('Content-Type'), 'application/lunch')
        self.assertEqual(body.read(), 'sandwiches')

        self.assertRaises(StopIteration, next, doc_iters)

 def test_get_response_headers_with_data(self):
     broker = backend.AccountBroker(':memory:', account='a')
     now = time.time()
     with mock.patch('time.time', new=lambda: now):
         broker.initialize(Timestamp(now).internal)
     # add some container data
     ts = (Timestamp(t).internal for t in itertools.count(int(now)))
     total_containers = 0
     total_objects = 0
     total_bytes = 0
     for policy in POLICIES:
         delete_timestamp = ts.next()
         put_timestamp = ts.next()
         object_count = int(policy)
         bytes_used = int(policy) * 10
         broker.put_container('c-%s' % policy.name, put_timestamp,
                              delete_timestamp, object_count, bytes_used,
                              int(policy))
         total_containers += 1
         total_objects += object_count
         total_bytes += bytes_used
     expected = HeaderKeyDict({
         'X-Account-Container-Count': total_containers,
         'X-Account-Object-Count': total_objects,
         'X-Account-Bytes-Used': total_bytes,
         'X-Timestamp': Timestamp(now).normal,
         'X-PUT-Timestamp': Timestamp(now).normal,
     })
     for policy in POLICIES:
         prefix = 'X-Account-Storage-Policy-%s-' % policy.name
         expected[prefix + 'Object-Count'] = int(policy)
         expected[prefix + 'Bytes-Used'] = int(policy) * 10
     resp_headers = utils.get_response_headers(broker)
     for key, value in resp_headers.items():
         expected_value = expected.pop(key)
         self.assertEqual(expected_value, str(value),
                          'value for %r was %r not %r' % (
                              key, value, expected_value))
     self.assertFalse(expected)