Python HTTPConnectionPool.urlopen方法代码示例

# 需要导入模块: from urllib3 import HTTPConnectionPool [as 别名]
# 或者: from urllib3.HTTPConnectionPool import urlopen [as 别名]
    def test_for_double_release(self):
        MAXSIZE=5

        # Check default state
        pool = HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE)
        self.assertEqual(pool.num_connections, 0)
        self.assertEqual(pool.pool.qsize(), MAXSIZE)

        # Make an empty slot for testing
        pool.pool.get()
        self.assertEqual(pool.pool.qsize(), MAXSIZE-1)

        # Check state after simple request
        pool.urlopen('GET', '/')
        self.assertEqual(pool.pool.qsize(), MAXSIZE-1)

        # Check state without release
        pool.urlopen('GET', '/', preload_content=False)
        self.assertEqual(pool.pool.qsize(), MAXSIZE-2)

        pool.urlopen('GET', '/')
        self.assertEqual(pool.pool.qsize(), MAXSIZE-2)

        # Check state after read
        pool.urlopen('GET', '/').data
        self.assertEqual(pool.pool.qsize(), MAXSIZE-2)

        pool.urlopen('GET', '/')
        self.assertEqual(pool.pool.qsize(), MAXSIZE-2)

# 需要导入模块: from urllib3 import HTTPConnectionPool [as 别名]
# 或者: from urllib3.HTTPConnectionPool import urlopen [as 别名]
class TestFileBodiesOnRetryOrRedirect(HTTPDummyServerTestCase):
    def setUp(self):
        self.pool = HTTPConnectionPool(self.host, self.port, timeout=0.1)
        self.addCleanup(self.pool.close)

    def test_retries_put_filehandle(self):
        """HTTP PUT retry with a file-like object should not timeout"""
        retry = Retry(total=3, status_forcelist=[418])
        # httplib reads in 8k chunks; use a larger content length
        content_length = 65535
        data = b'A' * content_length
        uploaded_file = io.BytesIO(data)
        headers = {'test-name': 'test_retries_put_filehandle',
                   'Content-Length': str(content_length)}
        resp = self.pool.urlopen('PUT', '/successful_retry',
                                 headers=headers,
                                 retries=retry,
                                 body=uploaded_file,
                                 assert_same_host=False, redirect=False)
        self.assertEqual(resp.status, 200)

    def test_redirect_put_file(self):
        """PUT with file object should work with a redirection response"""
        retry = Retry(total=3, status_forcelist=[418])
        # httplib reads in 8k chunks; use a larger content length
        content_length = 65535
        data = b'A' * content_length
        uploaded_file = io.BytesIO(data)
        headers = {'test-name': 'test_redirect_put_file',
                   'Content-Length': str(content_length)}
        url = '/redirect?target=/echo&status=307'
        resp = self.pool.urlopen('PUT', url,
                                 headers=headers,
                                 retries=retry,
                                 body=uploaded_file,
                                 assert_same_host=False, redirect=True)
        self.assertEqual(resp.status, 200)
        self.assertEqual(resp.data, data)

    def test_redirect_with_failed_tell(self):
        """Abort request if failed to get a position from tell()"""
        class BadTellObject(io.BytesIO):

            def tell(self):
                raise IOError

        body = BadTellObject(b'the data')
        url = '/redirect?target=/successful_retry'
        # httplib uses fileno if Content-Length isn't supplied,
        # which is unsupported by BytesIO.
        headers = {'Content-Length': '8'}
        try:
            self.pool.urlopen('PUT', url, headers=headers, body=body)
            self.fail('PUT successful despite failed rewind.')
        except UnrewindableBodyError as e:
            self.assertIn('Unable to record file position for', str(e))

# 需要导入模块: from urllib3 import HTTPConnectionPool [as 别名]
# 或者: from urllib3.HTTPConnectionPool import urlopen [as 别名]
    def test_provides_default_host_header(self):
        self.start_chunked_handler()
        chunks = ['foo', 'bar', '', 'bazzzzzzzzzzzzzzzzzzzzzz']
        pool = HTTPConnectionPool(self.host, self.port, retries=False)
        pool.urlopen('GET', '/', chunks, chunked=True)

        header_block = self.buffer.split(b'\r\n\r\n', 1)[0].lower()
        header_lines = header_block.split(b'\r\n')[1:]

        host_headers = [x for x in header_lines if x.startswith(b'host')]
        self.assertEqual(len(host_headers), 1)

# 需要导入模块: from urllib3 import HTTPConnectionPool [as 别名]
# 或者: from urllib3.HTTPConnectionPool import urlopen [as 别名]
 def test_conn_closed(self):
     pool = HTTPConnectionPool(self.host, self.port, timeout=0.001)
     conn = pool._get_conn()
     pool._put_conn(conn)
     try:
         url = '/sleep?seconds=0.005'
         pool.urlopen('GET', url)
         self.fail("The request should fail with a timeout error.")
     except ReadTimeoutError:
         if conn.sock:
             self.assertRaises(socket.error, conn.sock.recv, 1024)
     finally:
         pool._put_conn(conn)

# 需要导入模块: from urllib3 import HTTPConnectionPool [as 别名]
# 或者: from urllib3.HTTPConnectionPool import urlopen [as 别名]
    def test_removes_duplicate_host_header(self):
        self.start_chunked_handler()
        chunks = ['foo', 'bar', '', 'bazzzzzzzzzzzzzzzzzzzzzz']
        pool = HTTPConnectionPool(self.host, self.port, retries=False)
        self.addCleanup(pool.close)
        pool.urlopen(
            'GET', '/', chunks, headers={'Host': 'test.org'}, chunked=True
        )

        header_block = self.buffer.split(b'\r\n\r\n', 1)[0].lower()
        header_lines = header_block.split(b'\r\n')[1:]

        host_headers = [x for x in header_lines if x.startswith(b'host')]
        self.assertEqual(len(host_headers), 1)

# 需要导入模块: from urllib3 import HTTPConnectionPool [as 别名]
# 或者: from urllib3.HTTPConnectionPool import urlopen [as 别名]
class TestRetryPoolSize(HTTPDummyServerTestCase):
    def setUp(self):
        retries = Retry(
            total=1,
            raise_on_status=False,
            status_forcelist=[404],
        )
        self.pool = HTTPConnectionPool(self.host, self.port, maxsize=10,
                                       retries=retries, block=True)
        self.addCleanup(self.pool.close)

    def test_pool_size_retry(self):
        self.pool.urlopen('GET', '/not_found', preload_content=False)
        assert self.pool.num_connections == 1

# 需要导入模块: from urllib3 import HTTPConnectionPool [as 别名]
# 或者: from urllib3.HTTPConnectionPool import urlopen [as 别名]
    def test_chunks(self):
        self.start_chunked_handler()
        chunks = ['foo', 'bar', '', 'bazzzzzzzzzzzzzzzzzzzzzz']
        pool = HTTPConnectionPool(self.host, self.port, retries=False)
        pool.urlopen('GET', '/', chunks, headers=dict(DNT='1'), chunked=True)
        self.addCleanup(pool.close)

        self.assertIn(b'Transfer-Encoding', self.buffer)
        body = self.buffer.split(b'\r\n\r\n', 1)[1]
        lines = body.split(b'\r\n')
        # Empty chunks should have been skipped, as this could not be distinguished
        # from terminating the transmission
        for i, chunk in enumerate([c for c in chunks if c]):
            self.assertEqual(lines[i * 2], hex(len(chunk))[2:].encode('utf-8'))
            self.assertEqual(lines[i * 2 + 1], chunk.encode('utf-8'))

# 需要导入模块: from urllib3 import HTTPConnectionPool [as 别名]
# 或者: from urllib3.HTTPConnectionPool import urlopen [as 别名]
    def test_delayed_body_read_timeout(self):
        timed_out = Event()

        def socket_handler(listener):
            sock = listener.accept()[0]
            buf = b''
            body = 'Hi'
            while not buf.endswith(b'\r\n\r\n'):
                buf = sock.recv(65536)
            sock.send(('HTTP/1.1 200 OK\r\n'
                       'Content-Type: text/plain\r\n'
                       'Content-Length: %d\r\n'
                       '\r\n' % len(body)).encode('utf-8'))

            timed_out.wait()
            sock.send(body.encode('utf-8'))
            sock.close()

        self._start_server(socket_handler)
        pool = HTTPConnectionPool(self.host, self.port)

        response = pool.urlopen('GET', '/', retries=0, preload_content=False,
                                timeout=Timeout(connect=1, read=0.001))
        try:
            self.assertRaises(ReadTimeoutError, response.read)
        finally:
            timed_out.set()

# 需要导入模块: from urllib3 import HTTPConnectionPool [as 别名]
# 或者: from urllib3.HTTPConnectionPool import urlopen [as 别名]
    def test_conn_closed(self):
        block_event = Event()
        self.start_basic_handler(block_send=block_event, num=1)

        pool = HTTPConnectionPool(self.host, self.port, timeout=SHORT_TIMEOUT, retries=False)
        conn = pool._get_conn()
        pool._put_conn(conn)
        try:
            pool.urlopen('GET', '/')
            self.fail("The request should fail with a timeout error.")
        except ReadTimeoutError:
            if conn.sock:
                self.assertRaises(socket.error, conn.sock.recv, 1024)
        finally:
            pool._put_conn(conn)

        block_event.set()

# 需要导入模块: from urllib3 import HTTPConnectionPool [as 别名]
# 或者: from urllib3.HTTPConnectionPool import urlopen [as 别名]
    def _test_body(self, data):
        self.start_chunked_handler()
        pool = HTTPConnectionPool(self.host, self.port, retries=False)
        self.addCleanup(pool.close)

        pool.urlopen('GET', '/', data, chunked=True)
        header, body = self.buffer.split(b'\r\n\r\n', 1)

        self.assertIn(b'Transfer-Encoding: chunked', header.split(b'\r\n'))
        if data:
            bdata = data if isinstance(data, bytes) else data.encode('utf-8')
            self.assertIn(b'\r\n' + bdata + b'\r\n', body)
            self.assertTrue(body.endswith(b'\r\n0\r\n\r\n'))

            len_str = body.split(b'\r\n', 1)[0]
            stated_len = int(len_str, 16)
            self.assertEqual(stated_len, len(bdata))
        else:
            self.assertEqual(body, b'0\r\n\r\n')

# 需要导入模块: from urllib3 import HTTPConnectionPool [as 别名]
# 或者: from urllib3.HTTPConnectionPool import urlopen [as 别名]
	def download(self, url):
		""" download a url given in parameter
		"""
		http = HTTPConnectionPool(self.hostname)
		try:
			request = http.urlopen('GET', url)
			headerContent = request.headers.get('content-disposition')
			m = re.match('.*"(.*)".*', headerContent)
			filename = m.groups()[0]
			localfile = open('/'.join([self.outputDir, filename]), 'wb')
			localfile.write(request.data)
			localfile.close()
		except Exception, e:
			logging.exception(e)
			self.failed.append(url)

# 需要导入模块: from urllib3 import HTTPConnectionPool [as 别名]
# 或者: from urllib3.HTTPConnectionPool import urlopen [as 别名]
class deviceapi(object):
    def __init__(self,zenoss_server,zenoss_username,zenoss_password):
        self.zenoss_server=zenoss_server
        self.username=zenoss_username
        self.password=zenoss_password
    def conn(self):
        self.loginParams={'came_fraaaaaom':'http://'+self.zenoss_server+':8080/zport/dmd',
                     '__ac_name':self.username,
                     '__ac_password':self.password,
                     'submitted':'true'
                    }
        self.reqheaders={'Content-Type':'application/json'}
        self.reqCount = 1
        self.pool=HTTPConnectionPool(self.zenoss_server,port=8080,maxsize=5)
        self.loginResponse=self.pool.request('POST','/zport/acl_users/cookieAuthHelper/login',fields=self.loginParams,redirect=False)
        self.cookie={'cookie': self.loginResponse.getheader('set-cookie')}
        return self.cookie
    def operate(self,action,method,datalist=[],cookie={}):
            self.routers = {    'MessagingRouter': 'messaging',
                                'EventsRouter': 'evconsole',
                                'ProcessRouter': 'process',
                                'ServiceRouter': 'service',
                                'DeviceRouter': 'device',
                                'NetworkRouter': 'network',
                                'TemplateRouter': 'template',
                                'DetailNavRouter': 'detailnav',
                                'ReportRouter': 'report',
                                'MibRouter': 'mib',
                                'ZenPackRouter': 'zenpack' }

            self.cookie=cookie
            self.reqdata=[{
                            'type': 'rpc',
                            'data': datalist,
                            'method':method,
                            'action':action,
                            'tid':self.reqCount
                    }]
            self.reqCount +=1
            self.reqheaders.update(self.cookie)
            self.operateResponse=self.pool.urlopen('POST','/zport/dmd/'+self.routers[action]+'_router',body=json.dumps(self.reqdata),headers=self.reqheaders)
            if self.operateResponse.getheaders().getlist('Content-Type')[0] !='application/json':
                print('\033[1;31;47mLogin Failed, Please check your username and password !\033[0m')
                sys.exit(1)
            return self.operateResponse

# 需要导入模块: from urllib3 import HTTPConnectionPool [as 别名]
# 或者: from urllib3.HTTPConnectionPool import urlopen [as 别名]
class TestConnectionPool(HTTPDummyServerTestCase):

    def setUp(self):
        self.pool = HTTPConnectionPool(self.host, self.port)

    def test_get(self):
        r = self.pool.request('GET', '/specific_method',
                               fields={'method': 'GET'})
        self.assertEqual(r.status, 200, r.data)

    def test_post_url(self):
        r = self.pool.request('POST', '/specific_method',
                               fields={'method': 'POST'})
        self.assertEqual(r.status, 200, r.data)

    def test_urlopen_put(self):
        r = self.pool.urlopen('PUT', '/specific_method?method=PUT')
        self.assertEqual(r.status, 200, r.data)

    def test_wrong_specific_method(self):
        # To make sure the dummy server is actually returning failed responses
        r = self.pool.request('GET', '/specific_method',
                               fields={'method': 'POST'})
        self.assertEqual(r.status, 400, r.data)

        r = self.pool.request('POST', '/specific_method',
                               fields={'method': 'GET'})
        self.assertEqual(r.status, 400, r.data)

    def test_upload(self):
        data = "I'm in ur multipart form-data, hazing a cheezburgr"
        fields = {
            'upload_param': 'filefield',
            'upload_filename': 'lolcat.txt',
            'upload_size': len(data),
            'filefield': ('lolcat.txt', data),
        }

        r = self.pool.request('POST', '/upload', fields=fields)
        self.assertEqual(r.status, 200, r.data)

    def test_one_name_multiple_values(self):
        fields = [
            ('foo', 'a'),
            ('foo', 'b'),
        ]

        # urlencode
        r = self.pool.request('GET', '/echo', fields=fields)
        self.assertEqual(r.data, b'foo=a&foo=b')

        # multipart
        r = self.pool.request('POST', '/echo', fields=fields)
        self.assertEqual(r.data.count(b'name="foo"'), 2)


    def test_unicode_upload(self):
        fieldname = u('myfile')
        filename = u('\xe2\x99\xa5.txt')
        data = u('\xe2\x99\xa5').encode('utf8')
        size = len(data)

        fields = {
            u('upload_param'): fieldname,
            u('upload_filename'): filename,
            u('upload_size'): size,
            fieldname: (filename, data),
        }

        r = self.pool.request('POST', '/upload', fields=fields)
        self.assertEqual(r.status, 200, r.data)

    def test_timeout_float(self):
        url = '/sleep?seconds=0.005'
        # Pool-global timeout
        pool = HTTPConnectionPool(self.host, self.port, timeout=0.001)
        self.assertRaises(ReadTimeoutError, pool.request, 'GET', url)

    def test_conn_closed(self):
        pool = HTTPConnectionPool(self.host, self.port, timeout=0.001)
        conn = pool._get_conn()
        pool._put_conn(conn)
        try:
            url = '/sleep?seconds=0.005'
            pool.urlopen('GET', url)
            self.fail("The request should fail with a timeout error.")
        except ReadTimeoutError:
            if conn.sock:
                self.assertRaises(socket.error, conn.sock.recv, 1024)
        finally:
            pool._put_conn(conn)

    def test_nagle(self):
        """ Test that connections have TCP_NODELAY turned on """
        pool = HTTPConnectionPool(self.host, self.port)
        conn = pool._get_conn()
        pool._make_request(conn, 'GET', '/')
        tcp_nodelay_setting = conn.sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY)
        assert tcp_nodelay_setting > 0, ("Expected TCP_NODELAY to be set on the "
                                         "socket (with value greater than 0) "
#.........这里部分代码省略.........

# 需要导入模块: from urllib3 import HTTPConnectionPool [as 别名]
# 或者: from urllib3.HTTPConnectionPool import urlopen [as 别名]
class HttpClient(object):
    """
    Implements an mPlane HTTP client endpoint for component-push workflows. 
    This client endpoint can retrieve capabilities from a given URL, then post 
    Specifications to the component and retrieve Results or Receipts; it can
    also present Redeptions to retrieve Results.

    Caches retrieved Capabilities, Receipts, and Results.

    """
    def __init__(self, security, posturl, certfile=None):
        # store urls
        self._posturl = posturl
        url = urllib3.util.parse_url(posturl) 

        if security == True: 
            cert = mplane.utils.normalize_path(mplane.utils.read_setting(certfile, "cert"))
            key = mplane.utils.normalize_path(mplane.utils.read_setting(certfile, "key"))
            ca = mplane.utils.normalize_path(mplane.utils.read_setting(certfile, "ca-chain"))
            mplane.utils.check_file(cert)
            mplane.utils.check_file(key)
            mplane.utils.check_file(ca)
            self.pool = HTTPSConnectionPool(url.host, url.port, key_file=key, cert_file=cert, ca_certs=ca) 
        else: 
            self.pool = HTTPConnectionPool(url.host, url.port) 

        print("new client: "+self._posturl)

        # empty capability and measurement lists
        self._capabilities = OrderedDict()
        self._receipts = []
        self._results = []

    def get_mplane_reply(self, url, postmsg=None):
        """
        Given a URL, parses the object at the URL as an mPlane 
        message and processes it.

        Given a message to POST, sends the message to the given 
        URL and processes the reply as an mPlane message.

        """
        if postmsg is not None:
            print(postmsg)
            res = self.pool.urlopen('POST', url, 
                    body=postmsg.encode("utf-8"), 
                    headers={"content-type": "application/x-mplane+json"})
        else:
            res = self.pool.request('GET', url)
        print("get_mplane_reply "+url+" "+str(res.status)+" Content-Type "+res.getheader("content-type"))
        if res.status == 200 and \
           res.getheader("content-type") == "application/x-mplane+json":
            print("parsing json")
            return mplane.model.parse_json(res.data.decode("utf-8"))
        else:
            return [res.status, res.data.decode("utf-8")]

    def handle_message(self, msg, dn = None):
        """
        Processes a message. Caches capabilities, receipts, 
        and results, and handles Exceptions.

        """
        try:
            print("got message:")
            print(mplane.model.unparse_yaml(msg))

            if isinstance(msg, mplane.model.Capability):
                self.add_capability(msg, dn)
            elif isinstance(msg, mplane.model.Receipt):
                self.add_receipt(msg)
            elif isinstance(msg, mplane.model.Result):
                self.add_result(msg)
            elif isinstance(msg, mplane.model.Exception):
                self._handle_exception(msg)
            else:
                pass
        except:
            print("Supervisor returned: " + str(msg[0]) + " - " + msg[1])

    def add_capability(self, cap, dn):
        """Add a capability to the capability cache"""
        print("adding "+repr(cap))
        mplane.utils.add_value_to(self._capabilities, dn, cap)

    def clear_capabilities(self):
        """Clear the capability cache"""
        self._capabilities = OrderedDict()

    def retrieve_capabilities(self):
        """
        Given a URL, retrieves an object, parses it as an HTML page, 
        extracts links to capabilities, and retrieves and processes them
        into the capability cache.

        """
        self.clear_capabilities()
        url = "/" + S_CAPABILITY_PATH

        print("getting capabilities from " + url)
#.........这里部分代码省略.........

# 需要导入模块: from urllib3 import HTTPConnectionPool [as 别名]
# 或者: from urllib3.HTTPConnectionPool import urlopen [as 别名]
class TestConnectionPool(HTTPDummyServerTestCase):

    def setUp(self):
        self.pool = HTTPConnectionPool(self.host, self.port)

    def test_get(self):
        r = self.pool.request('GET', '/specific_method',
                               fields={'method': 'GET'})
        self.assertEqual(r.status, 200, r.data)

    def test_post_url(self):
        r = self.pool.request('POST', '/specific_method',
                               fields={'method': 'POST'})
        self.assertEqual(r.status, 200, r.data)

    def test_urlopen_put(self):
        r = self.pool.urlopen('PUT', '/specific_method?method=PUT')
        self.assertEqual(r.status, 200, r.data)

    def test_wrong_specific_method(self):
        # To make sure the dummy server is actually returning failed responses
        r = self.pool.request('GET', '/specific_method',
                               fields={'method': 'POST'})
        self.assertEqual(r.status, 400, r.data)

        r = self.pool.request('POST', '/specific_method',
                               fields={'method': 'GET'})
        self.assertEqual(r.status, 400, r.data)

    def test_upload(self):
        data = "I'm in ur multipart form-data, hazing a cheezburgr"
        fields = {
            'upload_param': 'filefield',
            'upload_filename': 'lolcat.txt',
            'upload_size': len(data),
            'filefield': ('lolcat.txt', data),
        }

        r = self.pool.request('POST', '/upload', fields=fields)
        self.assertEqual(r.status, 200, r.data)

    def test_one_name_multiple_values(self):
        fields = [
            ('foo', 'a'),
            ('foo', 'b'),
        ]

        # urlencode
        r = self.pool.request('GET', '/echo', fields=fields)
        self.assertEqual(r.data, b'foo=a&foo=b')

        # multipart
        r = self.pool.request('POST', '/echo', fields=fields)
        self.assertEqual(r.data.count(b'name="foo"'), 2)

    def test_request_method_body(self):
        body = b'hi'
        r = self.pool.request('POST', '/echo', body=body)
        self.assertEqual(r.data, body)

        fields = [('hi', 'hello')]
        self.assertRaises(TypeError, self.pool.request, 'POST', '/echo', body=body, fields=fields)

    def test_unicode_upload(self):
        fieldname = u('myfile')
        filename = u('\xe2\x99\xa5.txt')
        data = u('\xe2\x99\xa5').encode('utf8')
        size = len(data)

        fields = {
            u('upload_param'): fieldname,
            u('upload_filename'): filename,
            u('upload_size'): size,
            fieldname: (filename, data),
        }

        r = self.pool.request('POST', '/upload', fields=fields)
        self.assertEqual(r.status, 200, r.data)

    def test_nagle(self):
        """ Test that connections have TCP_NODELAY turned on """
        # This test needs to be here in order to be run. socket.create_connection actually tries to
        # connect to the host provided so we need a dummyserver to be running.
        pool = HTTPConnectionPool(self.host, self.port)
        conn = pool._get_conn()
        pool._make_request(conn, 'GET', '/')
        tcp_nodelay_setting = conn.sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY)
        self.assertTrue(tcp_nodelay_setting)

    def test_socket_options(self):
        """Test that connections accept socket options."""
        # This test needs to be here in order to be run. socket.create_connection actually tries to
        # connect to the host provided so we need a dummyserver to be running.
        pool = HTTPConnectionPool(self.host, self.port, socket_options=[
            (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
        ])
        s = pool._new_conn()._new_conn()  # Get the socket
        using_keepalive = s.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) > 0
        self.assertTrue(using_keepalive)
        s.close()
#.........这里部分代码省略.........