Python bufferedhttp.http_connect函数代码示例

    def update(self, job):
        """
        High-level method that replicates a single partition.

        :param job: a dict containing info about the partition to be replicated
        """
        self.replication_count += 1
        begin = time.time()
        try:
            hashed, local_hash = get_hashes(job['path'],
                    do_listdir=(self.replication_count % 10) == 0,
                    reclaim_age=self.reclaim_age)
            self.suffix_hash += hashed
            attempts_left = self.object_ring.replica_count - 1
            nodes = itertools.chain(job['nodes'],
                        self.object_ring.get_more_nodes(int(job['partition'])))
            while attempts_left > 0:
                # If this throws StopIterator it will be caught way below
                node = next(nodes)
                attempts_left -= 1
                try:
                    with Timeout(self.http_timeout):
                        resp = http_connect(node['ip'], node['port'],
                                node['device'], job['partition'], 'REPLICATE',
                            '', headers={'Content-Length': '0'}).getresponse()
                        if resp.status == 507:
                            self.logger.error(_('%(ip)s/%(device)s responded'
                                    ' as unmounted'), node)
                            attempts_left += 1
                            continue
                        if resp.status != 200:
                            self.logger.error(_("Invalid response %(resp)s "
                                "from %(ip)s"),
                                {'resp': resp.status, 'ip': node['ip']})
                            continue
                        remote_hash = pickle.loads(resp.read())
                        del resp
                    suffixes = [suffix for suffix in local_hash
                                  if local_hash[suffix] !=
                                     remote_hash.get(suffix, -1)]
                    if not suffixes:
                        continue
                    self.rsync(node, job, suffixes)
                    recalculate_hashes(job['path'], suffixes,
                                       reclaim_age=self.reclaim_age)
                    with Timeout(self.http_timeout):
                        conn = http_connect(node['ip'], node['port'],
                            node['device'], job['partition'], 'REPLICATE',
                            '/' + '-'.join(suffixes),
                            headers={'Content-Length': '0'})
                        conn.getresponse().read()
                    self.suffix_sync += len(suffixes)
                except (Exception, Timeout):
                    self.logger.exception(_("Error syncing with node: %s") %
                                            node)
            self.suffix_count += len(local_hash)
        except (Exception, Timeout):
            self.logger.exception(_("Error syncing partition"))
        finally:
            self.partition_times.append(time.time() - begin)

    def update_deleted(self, job):
        """
        High-level method that replicates a single partition that doesn't
        belong on this node.

        :param job: a dict containing info about the partition to be replicated
        """

        def tpool_get_suffixes(path):
            return [suff for suff in os.listdir(path)
                    if len(suff) == 3 and isdir(join(path, suff))]
        self.replication_count += 1
        begin = time.time()
        try:
            responses = []
            suffixes = tpool.execute(tpool_get_suffixes, job['path'])
            if suffixes:
                for node in job['nodes']:
                    success = self.rsync(node, job, suffixes)
                    if success:
                        with Timeout(self.http_timeout):
                            http_connect(node['ip'], node['port'],
                                node['device'], job['partition'], 'REPLICATE',
                                '/' + '-'.join(suffixes),
                          headers={'Content-Length': '0'}).getresponse().read()
                    responses.append(success)
            if not suffixes or (len(responses) == \
                        self.object_ring.replica_count and all(responses)):
                self.logger.info(_("Removing partition: %s"), job['path'])
                tpool.execute(shutil.rmtree, job['path'], ignore_errors=True)
        except (Exception, Timeout):
            self.logger.exception(_("Error syncing handoff partition"))
        finally:
            self.partition_times.append(time.time() - begin)

    def test_nonstr_header_values(self):

        class MockHTTPSConnection(object):

            def __init__(self, hostport):
                pass

            def putrequest(self, method, path, skip_host=0):
                pass

            def putheader(self, header, *values):
                # Essentially what Python 2.7 does that caused us problems.
                '\r\n\t'.join(values)

            def endheaders(self):
                pass

        origHTTPSConnection = bufferedhttp.HTTPSConnection
        bufferedhttp.HTTPSConnection = MockHTTPSConnection
        try:
            bufferedhttp.http_connect('127.0.0.1', 8080, 'sda', 1, 'GET', '/',
                headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0,
                         'x-four': {'crazy': 'value'}}, ssl=True)
            bufferedhttp.http_connect_raw('127.0.0.1', 8080, 'GET', '/',
                headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0,
                         'x-four': {'crazy': 'value'}}, ssl=True)
        finally:
            bufferedhttp.HTTPSConnection = origHTTPSConnection

 def test_nonstr_header_values(self):
     with mock.patch('swift.common.bufferedhttp.HTTPSConnection',
                     MockHTTPSConnection):
         bufferedhttp.http_connect(
             '127.0.0.1', 8080, 'sda', 1, 'GET', '/',
             headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0,
                      'x-four': {'crazy': 'value'}}, ssl=True)
         bufferedhttp.http_connect_raw(
             '127.0.0.1', 8080, 'GET', '/',
             headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0,
                      'x-four': {'crazy': 'value'}}, ssl=True)

 def test_nonstr_header_values(self):
     origHTTPSConnection = bufferedhttp.HTTPSConnection
     bufferedhttp.HTTPSConnection = MockHTTPSConnection
     try:
         bufferedhttp.http_connect(
             '127.0.0.1', 8080, 'sda', 1, 'GET', '/',
             headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0,
                      'x-four': {'crazy': 'value'}}, ssl=True)
         bufferedhttp.http_connect_raw(
             '127.0.0.1', 8080, 'GET', '/',
             headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0,
                      'x-four': {'crazy': 'value'}}, ssl=True)
     finally:
         bufferedhttp.HTTPSConnection = origHTTPSConnection

def direct_delete_object(node, part, account, container, obj,
        conn_timeout=5, response_timeout=15, headers={}):
    """
    Delete object directly from the object server.

    :param node: node dictionary from the ring
    :param part: partition the container is on
    :param account: account name
    :param container: container name
    :param obj: object name
    :param conn_timeout: timeout in seconds for establishing the connection
    :param response_timeout: timeout in seconds for getting the response
    :returns: response from server
    """
    path = '/%s/%s/%s' % (account, container, obj)
    headers['X-Timestamp'] = normalize_timestamp(time())
    with Timeout(conn_timeout):
        conn = http_connect(node['ip'], node['port'], node['device'], part,
                'DELETE', path, headers)
    with Timeout(response_timeout):
        resp = conn.getresponse()
        resp.read()
    if not is_success(resp.status):
        raise ClientException(
                'Object server %s:%s direct DELETE %s gave status %s' %
                (node['ip'], node['port'],
                repr('/%s/%s%s' % (node['device'], part, path)),
                resp.status),
                http_host=node['ip'], http_port=node['port'],
                http_device=node['device'], http_status=resp.status,
                http_reason=resp.reason)

 def _connect_put_node(self, nodes, part, path, headers,
                       logger_thread_locals):
     """Method for a file PUT connect"""
     self.app.logger.thread_locals = logger_thread_locals
     for node in nodes:
         try:
             start_time = time.time()
             with ConnectionTimeout(self.app.conn_timeout):
                 conn = http_connect(
                     node['ip'], node['port'], node['device'], part, 'PUT',
                     path, headers)
             self.app.set_node_timing(node, time.time() - start_time)
             with Timeout(self.app.node_timeout):
                 resp = conn.getexpect()
             if resp.status == HTTP_CONTINUE:
                 conn.resp = None
                 conn.node = node
                 return conn
             elif is_success(resp.status):
                 conn.resp = resp
                 conn.node = node
                 return conn
             elif resp.status == HTTP_INSUFFICIENT_STORAGE:
                 self.error_limit(node)
         except:
             self.exception_occurred(node, _('Object'),
                                     _('Expect: 100-continue on %s') % path)

def direct_head_object(node, part, account, container, obj, conn_timeout=5,
                       response_timeout=15, headers=None):
    """
    Request object information directly from the object server.

    :param node: node dictionary from the ring
    :param part: partition the container is on
    :param account: account name
    :param container: container name
    :param obj: object name
    :param conn_timeout: timeout in seconds for establishing the connection
    :param response_timeout: timeout in seconds for getting the response
    :param headers: dict to be passed into HTTPConnection headers
    :returns: a dict containing the response's headers in a HeaderKeyDict
    """
    if headers is None:
        headers = {}

    headers = gen_headers(headers)

    path = '/%s/%s/%s' % (account, container, obj)
    with Timeout(conn_timeout):
        conn = http_connect(node['ip'], node['port'], node['device'], part,
                            'HEAD', path, headers=headers)
    with Timeout(response_timeout):
        resp = conn.getresponse()
        resp.read()
    if not is_success(resp.status):
        raise DirectClientException('Object', 'HEAD',
                                    node, part, path, resp)
    resp_headers = HeaderKeyDict()
    for header, value in resp.getheaders():
        resp_headers[header] = value
    return resp_headers

def direct_post_object(node, part, account, container, name, headers,
                       conn_timeout=5, response_timeout=15):
    """
    Direct update to object metadata on object server.

    :param node: node dictionary from the ring
    :param part: partition the container is on
    :param account: account name
    :param container: container name
    :param name: object name
    :param headers: headers to store as metadata
    :param conn_timeout: timeout in seconds for establishing the connection
    :param response_timeout: timeout in seconds for getting the response
    :raises ClientException: HTTP POST request failed
    """
    path = '/%s/%s/%s' % (account, container, name)
    with Timeout(conn_timeout):
        conn = http_connect(node['ip'], node['port'], node['device'], part,
                            'POST', path, headers=gen_headers(headers, True))
    with Timeout(response_timeout):
        resp = conn.getresponse()
        resp.read()
    if not is_success(resp.status):
        raise DirectClientException('Object', 'POST',
                                    node, part, path, resp)

def _get_direct_account_container(path, stype, node, part,
                                  marker=None, limit=None,
                                  prefix=None, delimiter=None, conn_timeout=5,
                                  response_timeout=15):
    """Base class for get direct account and container.

    Do not use directly use the get_direct_account or
    get_direct_container instead.
    """
    qs = 'format=json'
    if marker:
        qs += '&marker=%s' % quote(marker)
    if limit:
        qs += '&limit=%d' % limit
    if prefix:
        qs += '&prefix=%s' % quote(prefix)
    if delimiter:
        qs += '&delimiter=%s' % quote(delimiter)
    with Timeout(conn_timeout):
        conn = http_connect(node['ip'], node['port'], node['device'], part,
                            'GET', path, query_string=qs,
                            headers=gen_headers())
    with Timeout(response_timeout):
        resp = conn.getresponse()
    if not is_success(resp.status):
        resp.read()
        raise DirectClientException(stype, 'GET', node, part, path, resp)

    resp_headers = HeaderKeyDict()
    for header, value in resp.getheaders():
        resp_headers[header] = value
    if resp.status == HTTP_NO_CONTENT:
        resp.read()
        return resp_headers, []
    return resp_headers, json.loads(resp.read())

def direct_head_container(node, part, account, container, conn_timeout=5,
                          response_timeout=15):
    """
    Request container information directly from the container server.

    :param node: node dictionary from the ring
    :param part: partition the container is on
    :param account: account name
    :param container: container name
    :param conn_timeout: timeout in seconds for establishing the connection
    :param response_timeout: timeout in seconds for getting the response
    :returns: a dict containing the response's headers in a HeaderKeyDict
    """
    path = '/%s/%s' % (account, container)
    with Timeout(conn_timeout):
        conn = http_connect(node['ip'], node['port'], node['device'], part,
                            'HEAD', path, headers=gen_headers())
    with Timeout(response_timeout):
        resp = conn.getresponse()
        resp.read()
    if not is_success(resp.status):
        raise DirectClientException('Container', 'HEAD',
                                    node, part, path, resp)
    resp_headers = HeaderKeyDict()
    for header, value in resp.getheaders():
        resp_headers[header] = value
    return resp_headers

def direct_post_object(node, part, account, container, name, headers,
                       conn_timeout=5, response_timeout=15):
    """
    Direct update to object metadata on object server.

    :param node: node dictionary from the ring
    :param part: partition the container is on
    :param account: account name
    :param container: container name
    :param name: object name
    :param headers: headers to store as metadata
    :param conn_timeout: timeout in seconds for establishing the connection
    :param response_timeout: timeout in seconds for getting the response
    :raises ClientException: HTTP POST request failed
    """
    path = '/%s/%s/%s' % (account, container, name)
    headers['X-Timestamp'] = normalize_timestamp(time())
    with Timeout(conn_timeout):
        conn = http_connect(node['ip'], node['port'], node['device'], part,
                'POST', path, headers=headers)
    with Timeout(response_timeout):
        resp = conn.getresponse()
        resp.read()
    if not is_success(resp.status):
        raise ClientException(
                'Object server %s:%s direct POST %s gave status %s' %
                (node['ip'], node['port'],
                repr('/%s/%s%s' % (node['device'], part, path)),
                resp.status),
                http_host=node['ip'], http_port=node['port'],
                http_device=node['device'], http_status=resp.status,
                http_reason=resp.reason)

    def get_object(self, account, container, object):
        """
        GET object from a standalone node
        """
        
        node = {'zone': 3, 'weight': 100.0, 'ip': self.address, 
                'id': 3, 'meta': '', 'device': 'sda6', 'port': self.port}
        
        headers = dict()
        partition = self.__get_partition__(account, container, object, self.part_shift)
        
        path = "/%s/%s/%s" % (account, container, object)
        method = "GET"
        conn = http_connect(node['ip'], node['port'],#class
                        node['device'], partition, method, path,
                        #headers=headers,
                        #query_string=''
                        )
        
        resp = conn.getresponse()
        status = resp.status
        headers = resp.getheaders()
        content = resp.read()

        return (status, headers, content)#http's return value, headers contain more information and could be verified later, content is the file content.

def direct_get_suffix_hashes(node, part, suffixes, conn_timeout=5,
                             response_timeout=15, headers=None):
    """
    Get suffix hashes directly from the object server.

    :param node: node dictionary from the ring
    :param part: partition the container is on
    :param conn_timeout: timeout in seconds for establishing the connection
    :param response_timeout: timeout in seconds for getting the response
    :param headers: dict to be passed into HTTPConnection headers
    :returns: dict of suffix hashes
    :raises ClientException: HTTP REPLICATE request failed
    """
    if headers is None:
        headers = {}

    path = '/%s' % '-'.join(suffixes)
    with Timeout(conn_timeout):
        conn = http_connect(node['replication_ip'], node['replication_port'],
                            node['device'], part, 'REPLICATE', path,
                            headers=gen_headers(headers))
    with Timeout(response_timeout):
        resp = conn.getresponse()
    if not is_success(resp.status):
        raise DirectClientException('Object', 'REPLICATE',
                                    node, part, path, resp,
                                    host={'ip': node['replication_ip'],
                                          'port': node['replication_port']}
                                    )
    return pickle.loads(resp.read())

    def object_update(self, node, part, op, obj, headers_out):
        """
        Perform the object update to the container

        :param node: node dictionary from the container ring
        :param part: partition that holds the container
        :param op: operation performed (ex: 'PUT' or 'DELETE')
        :param obj: object name being updated
        :param headers_out: headers to send with the update
        """
        try:
            with ConnectionTimeout(self.conn_timeout):
                conn = http_connect(node['ip'], node['port'], node['device'],
                                    part, op, obj, headers_out)
            with Timeout(self.node_timeout):
                resp = conn.getresponse()
                resp.read()
                success = is_success(resp.status)
                if not success:
                    self.logger.debug(
                        _('Error code %(status)d is returned from remote '
                          'server %(ip)s: %(port)s / %(device)s'),
                        {'status': resp.status, 'ip': node['ip'],
                         'port': node['port'], 'device': node['device']})
                return (success, node['id'])
        except (Exception, Timeout):
            self.logger.exception(_('ERROR with remote server '
                                    '%(ip)s:%(port)s/%(device)s'), node)
        return HTTP_INTERNAL_SERVER_ERROR, node['id']