Python AsyncHTTPClient.configure方法代码示例

# 需要导入模块: from tornado.httpclient import AsyncHTTPClient [as 别名]
# 或者: from tornado.httpclient.AsyncHTTPClient import configure [as 别名]
def tcount_exception(counter_name):
    exc_name, exc_value, lineno, lines = get_exception_info()
    data = generate_json_record(counter_name, exc_name, exc_value, lineno, lines)
    # configure to use the better fetch method
    AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
    # make the actual call to the counter
    AsyncHTTPClient().fetch(URL, lambda resp: None, method="POST", body=json.dumps(data))

# 需要导入模块: from tornado.httpclient import AsyncHTTPClient [as 别名]
# 或者: from tornado.httpclient.AsyncHTTPClient import configure [as 别名]
 def __init__(self, host = 'localhost', max_clients=100, port = 4711, logger = None):
     AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
     self.http_client = AsyncHTTPClient(max_clients=max_clients)
     self.host = host
     self.port = port
     self.logger = logger
     self.body = ''

# 需要导入模块: from tornado.httpclient import AsyncHTTPClient [as 别名]
# 或者: from tornado.httpclient.AsyncHTTPClient import configure [as 别名]
    def __init__(self, url, callback = None, retry = 0, keep_alive = False, ssl = False, validate_cert = False, user = None, password = None):
        """
        Build the event source client
        :param url: string, the url to connect to
        :param action: string of the listening action to connect to
        :param target: string with the listening token
        :param callback: function with one parameter (Event) that gets called for each received event
        :param retry: timeout between two reconnections (0 means no reconnection)
        """
        log.debug("EventSourceClient(%s,%s,%s)" % (url, callback, retry))

        self.data_partial = None
        self.last_event_id = None
        self.retry_timeout = int(retry)
        self.keep_alive = keep_alive
        self._url = url
        self._headers = {"Accept": "text/event-stream"}
        self._user = user
        self._password = password

        AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
        self.http_client = AsyncHTTPClient()
        self.http_request = HTTPRequest(url = self._url,
                                        method="GET",
                                        headers={"content-type":"text/event-stream"},
                                        request_timeout = 0,
                                        validate_cert = validate_cert,
                                        streaming_callback = self.handle_stream,
                                        auth_username = user,
                                        auth_password = password)
        if callback is None:
            self.cb = lambda e: log.info( "received %s" % (e,) )
        else:
            self.cb = callback

# 需要导入模块: from tornado.httpclient import AsyncHTTPClient [as 别名]
# 或者: from tornado.httpclient.AsyncHTTPClient import configure [as 别名]
    def __init__(
            self, concurrency=10, auto_start=False, cache=False,
            expiration_in_seconds=30, request_timeout_in_seconds=10,
            connect_timeout_in_seconds=5, ignore_pycurl=False,
            limiter=None, allow_connection_reuse=True):

        self.concurrency = concurrency
        self.auto_start = auto_start
        self.last_timeout = None

        self.cache = cache
        self.response_cache = Cache(expiration_in_seconds=expiration_in_seconds)
        self.request_timeout_in_seconds = request_timeout_in_seconds
        self.connect_timeout_in_seconds = connect_timeout_in_seconds

        self.ignore_pycurl = ignore_pycurl

        self.running_urls = 0
        self.url_queue = []

        if PYCURL_AVAILABLE and not self.ignore_pycurl:
            logging.debug('pycurl is available, thus Octopus will be using it instead of tornado\'s simple http client.')
            AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
            self.allow_connection_reuse = allow_connection_reuse
        else:
            self.allow_connection_reuse = True

        if auto_start:
            logging.debug('Auto starting...')
            self.start()

        self.limiter = limiter

# 需要导入模块: from tornado.httpclient import AsyncHTTPClient [as 别名]
# 或者: from tornado.httpclient.AsyncHTTPClient import configure [as 别名]
    def __init__(self):
        AsyncHTTPClient.configure(
            "tornado.curl_httpclient.CurlAsyncHTTPClient")
        self.client = AsyncHTTPClient()

        # 开发者 token
        self.client_id = WEI_CLIENT_ID
        self.client_secret = WEI_CLIENT_SECRET
        self.admin_token = WEI_ADMIN_TOKEN

        # oauth 相关 url
        self.oauth2_url = 'https://api.weibo.com/oauth2/authorize?' \
                          'client_id=%s&' \
                          'response_type=code&' \
                          'redirect_uri=twiwei.com/oauth2/weibo/access_token/' % self.client_id
        self.access_token_url = 'https://api.weibo.com/oauth2/access_token'

        # 时间线相关 url
        self.user_home_url = 'https://api.weibo.com/2/statuses/home_timeline.json'
        self.public_url = 'https://api.weibo.com/2/statuses/public_timeline.json'

        # 用户操作相关 url
        self.like_weibo_url = 'https://api.weibo.com/2/favorites/create.json'
        self.unlike_weibo_url = 'https://api.weibo.com/2/favorites/destroy.json'
        self.get_replies_url = 'https://api.weibo.com/2/comments/show.json'
        self.repost_message_url = 'https://api.weibo.com/2/statuses/repost.json'
        self.reply_message_url = 'https://api.weibo.com/2/comments/create.json'

        # 用户信息相关 url
        self.user_info_url = 'https://api.weibo.com/2/users/show.json'
        self.liked_url = 'https://api.weibo.com/2/favorites/ids.json'

# 需要导入模块: from tornado.httpclient import AsyncHTTPClient [as 别名]
# 或者: from tornado.httpclient.AsyncHTTPClient import configure [as 别名]
    def test_max_clients(self):
        # The max_clients argument is tricky because it was originally
        # allowed to be passed positionally; newer arguments are keyword-only.
        AsyncHTTPClient.configure(SimpleAsyncHTTPClient)
        with closing(AsyncHTTPClient(
                self.io_loop, force_instance=True)) as client:
            self.assertEqual(client.max_clients, 10)
        with closing(AsyncHTTPClient(
                self.io_loop, 11, force_instance=True)) as client:
            self.assertEqual(client.max_clients, 11)
        with closing(AsyncHTTPClient(
                self.io_loop, max_clients=11, force_instance=True)) as client:
            self.assertEqual(client.max_clients, 11)

        # Now configure max_clients statically and try overriding it
        # with each way max_clients can be passed
        AsyncHTTPClient.configure(SimpleAsyncHTTPClient, max_clients=12)
        with closing(AsyncHTTPClient(
                self.io_loop, force_instance=True)) as client:
            self.assertEqual(client.max_clients, 12)
        with closing(AsyncHTTPClient(
                self.io_loop, max_clients=13, force_instance=True)) as client:
            self.assertEqual(client.max_clients, 13)
        with closing(AsyncHTTPClient(
                self.io_loop, max_clients=14, force_instance=True)) as client:
            self.assertEqual(client.max_clients, 14)

# 需要导入模块: from tornado.httpclient import AsyncHTTPClient [as 别名]
# 或者: from tornado.httpclient.AsyncHTTPClient import configure [as 别名]
def http_request(url, server, port=80, timeout=20.0):
    def check_twitter_response(response):
        return (response is not None and len(response.headers.get_list('Server')) > 0 \
                and response.headers.get_list('Server')[0] == 'tfe')

    def get_rate_limits(headers):
        rtime = headers.get_list('X-RateLimit-Reset')
        rhits = headers.get_list('X-RateLimit-Remaining')
        if len(rtime) > 0: rtime = int(rtime[0])
        else: rtime = None
        if len(rhits) > 0: rhits = int(rhits[0])
        else: rhits = None
        return rhits, rtime

    AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
    http_client = HTTPClient()
    code, data, rtime, rhits = 999, None, None, None
    try:
        response = http_client.fetch(url, proxy_host=server, proxy_port=port, connect_timeout=timeout, request_timeout=timeout)
        response.rethrow()

        if check_twitter_response(response):
            code, data = response.code, response.body
            rhits, rtime = get_rate_limits(response.headers)

    except HTTPError as e:
        if check_twitter_response(e.response):
            code, data = e.code, None
            rhits, rtime = get_rate_limits(e.response.headers)

    return code, data, rhits, rtime

# 需要导入模块: from tornado.httpclient import AsyncHTTPClient [as 别名]
# 或者: from tornado.httpclient.AsyncHTTPClient import configure [as 别名]
 def __init__(self,url,action,target,callback=None,retry=0,ssl=False):
     """
     Build the event source client
     :param url: string, the url to connect to
     :param action: string of the listening action to connect to
     :param target: string with the listening token
     :param callback: function with one parameter (Event) that gets called for each received event
     :param retry: timeout between two reconnections (0 means no reconnection)
     """
     log.debug("EventSourceClient(%s,%s,%s,%s,%s)" % (url,action,target,callback,retry))
     
     if ssl:
         self._url = "https://%s/%s/%s" % (url,action,target)
     else:
         self._url = "http://%s/%s/%s" % (url,action,target)
     AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")
     self.http_client = AsyncHTTPClient()
     self.http_request = HTTPRequest(url=self._url,
                                     method='GET',
                                     headers={"content-type":"text/event-stream"},
                                     request_timeout=0,
                                     streaming_callback=self.handle_stream)
     if callback is None:
         self.cb = lambda e: log.info( "received %s" % (e,) )
     else:
         self.cb = callback
     self.retry_timeout = int(retry)

# 需要导入模块: from tornado.httpclient import AsyncHTTPClient [as 别名]
# 或者: from tornado.httpclient.AsyncHTTPClient import configure [as 别名]
    def tsend_log(self,
            message,
            severity,
            filename=None,
            url=None,
            status_code=None,
            headers=None,
            parameters=None,
            stacktrace=False):
        from tornado.httpclient import AsyncHTTPClient

        d = self._build_message(
                message,
                severity,
                filename,
                url,
                status_code,
                headers,
                parameters,
                stacktrace)

        # want to use the better client here.
        AsyncHTTPClient.configure("tornado.curl_httpclient.CurlAsyncHTTPClient")

        AsyncHTTPClient().fetch(
                self._build_url(),
                lambda resp: None,
                method="POST",
                body=json.dumps(d),
                headers=headers)

# 需要导入模块: from tornado.httpclient import AsyncHTTPClient [as 别名]
# 或者: from tornado.httpclient.AsyncHTTPClient import configure [as 别名]
    def __init__(self, url, callback = None, retry = 0, keep_alive = False, ssl = False, validate_cert = False, user = None, password = None, async_http_client = DEFAULT_ASYNC_HTTP_CLIENT):
        """
        Build the event source client
        :param url: string, the url to connect to
        :param action: string of the listening action to connect to
        :param target: string with the listening token
        :param callback: function with one parameter (Event) that gets called for each received event
        :param retry: timeout between two reconnections (0 means no reconnection)
        """
        log.debug("EventSourceClient(%s,%s,%s)" % (url, callback, retry))

        self.data_partial = None
        self.last_event_id = None
        self.retry_timeout = int(retry)
        self.keep_alive = keep_alive
        self.current_event = None
        self._url = url
        self._headers = {"Accept": "text/event-stream"}
        self._options = {
            'validate_cert': validate_cert,
            'auth_username': user,
            'auth_password': password
        }

        if async_http_client is not None:
            AsyncHTTPClient.configure(async_http_client)
        self.http_client = AsyncHTTPClient()
        if callback is None:
            self.cb = lambda e: log.info( "received %s" % (e,) )
        else:
            self.cb = callback

# 需要导入模块: from tornado.httpclient import AsyncHTTPClient [as 别名]
# 或者: from tornado.httpclient.AsyncHTTPClient import configure [as 别名]
def getAClient(max_clients=200):
	client = AsyncHTTPClient(max_clients=max_clients)  # page 20 * size 20,maybe great
	client.configure(None,
	                 # "tornado.curl_httpclient.CurlAsyncHTTPClient",
	                 raise_error=False
	                 )
	return client

# 需要导入模块: from tornado.httpclient import AsyncHTTPClient [as 别名]
# 或者: from tornado.httpclient.AsyncHTTPClient import configure [as 别名]
def main():

    term, location, search_limit = 'dinner', 'Chattanooga, TN', 3
    url_params = {
        'term': term.replace(' ', '+'),
        'location': location.replace(' ','+'),
        'limit' : search_limit
    }

    def on_response(response, error):
        ioloop.IOLoop.instance().stop()

        if error:
            raise error

        for repo in response:
            print repo

    auth = OAuth1(client_key=CLIENT_KEY,
                  client_secret=CLIENT_SECRET,
                  resource_owner_key=RESOURCE_OWNER_KEY,
                  resource_owner_secret=RESOURCE_OWNER_SECRET,
    )

    AsyncHTTPClient.configure(None, defaults=dict(user_agent="ipa-agent"))
    buss = Businesses(AsyncHTTPClient())
    buss.all(on_response, auth=auth, url_params=url_params)

    ioloop.IOLoop.instance().start()

# 需要导入模块: from tornado.httpclient import AsyncHTTPClient [as 别名]
# 或者: from tornado.httpclient.AsyncHTTPClient import configure [as 别名]
def main():
    def scrapy(body):
        bs = BeautifulSoup(body)
        print(bs)
        item = bs.find("div",id="product-intro").find("div",id="itemInfo")
        name = item.find("div",id="name").find("h1").text
        price = item.find("div",id="summary-price").find("strong",class_="p-price").text
        #print(price)
        price = re_price.search(price).group()
        discount = item.find("div",id="summary-price").find("span",class_="p-discount").text
        #print(discount)
        discount = re_price.search(discount).group()
        #print(name,price,discount)
        print("insert a shop(%s,%s,%s)"%(name,price,discount))



    def prepare_cul_opts(obj):
        pass
        #obj.setopt(pycurl.WRITEFUNCTION,open("result.html","wb").write)
        #obj.setopt(pycurl.NOBODY,True)
    #url="http://upload-images.jianshu.io/upload_images/1679702-7e810a34f3ef8d18.jpg?imageMogr2/auto-orient/strip%7CimageView2/1/w/300/h/300"
    AsyncHTTPClient.configure(CurlAsyncHTTPClient)
    for url in ["http://item.jd.com/11917788.html"]:
        httpCli = AsyncHTTPClient()
        respone = yield httpCli.fetch(url,prepare_curl_callback=prepare_cul_opts)
        scrapy(respone.body)
        print(list(respone.headers.get_all()))

# 需要导入模块: from tornado.httpclient import AsyncHTTPClient [as 别名]
# 或者: from tornado.httpclient.AsyncHTTPClient import configure [as 别名]
def client(io_loop, request):
    """Return mocked AsyncHTTPClient"""
    before = AsyncHTTPClient.configured_class()
    AsyncHTTPClient.configure(MockAsyncHTTPClient)
    request.addfinalizer(lambda : AsyncHTTPClient.configure(before))
    c = AsyncHTTPClient()
    assert isinstance(c, MockAsyncHTTPClient)
    return c

# 需要导入模块: from tornado.httpclient import AsyncHTTPClient [as 别名]
# 或者: from tornado.httpclient.AsyncHTTPClient import configure [as 别名]
    def get_http_client(self):
        """Overrides `AsyncHTTPTestCase.get_http_client` to separate unit test HTTPClient
        from application HTTPClient.

        This allows mocking HTTP requests made by application in unit tests.
        """
        AsyncHTTPClient.configure('tornado.curl_httpclient.CurlAsyncHTTPClient')
        return AsyncHTTPClient(force_instance=True)