Python CurlAsyncHTTPClient.fetch方法代码示例

# 需要导入模块: from tornado.curl_httpclient import CurlAsyncHTTPClient [as 别名]
# 或者: from tornado.curl_httpclient.CurlAsyncHTTPClient import fetch [as 别名]
def del_media_group(local_config, group, media_type, callback):
    scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
    
    logging.debug('deleting group "%(group)s" of remote camera %(id)s on %(url)s' % {
            'group': group or 'ungrouped',
            'id': camera_id,
            'url': pretty_camera_url(local_config)})
    
    path += '/%(media_type)s/%(id)s/delete_all/%(group)s/' % {
            'media_type': media_type,
            'id': camera_id,
            'group': group}

    request = _make_request(scheme, host, port, username, password, path,
            method='POST', data='{}',
            timeout=settings.REMOTE_REQUEST_TIMEOUT, content_type='application/json')

    def on_response(response):
        if response.error:
            logging.error('failed to delete group "%(group)s" of remote camera %(id)s on %(url)s: %(msg)s' % {
                    'group': group or 'ungrouped',
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'msg': utils.pretty_http_error(response)})
            
            return callback(error=utils.pretty_http_error(response))
        
        callback()

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

# 需要导入模块: from tornado.curl_httpclient import CurlAsyncHTTPClient [as 别名]
# 或者: from tornado.curl_httpclient.CurlAsyncHTTPClient import fetch [as 别名]
class CurlHTTPClientTestCase(AsyncHTTPTestCase):
	def setUp(self):
		super(CurlHTTPClientTestCase, self).setUp()
		self.http_client = CurlAsyncHTTPClient(self.io_loop)

	def get_app(self):
		return Application([
			('/digest', DigestAuthHandler),
		])

	def test_prepare_curl_callback_stack_context(self):
		exc_info = []

		def error_handler(typ, value, tb):
			exc_info.append((typ, value, tb))
			self.stop()
			return True

		with ExceptionStackContext(error_handler):
			request = HTTPRequest(self.get_url('/'),
								  prepare_curl_callback=lambda curl: 1 / 0)
		self.http_client.fetch(request, callback=self.stop)
		self.wait()
		self.assertEqual(1, len(exc_info))
		self.assertIs(exc_info[0][0], ZeroDivisionError)

	def test_digest_auth(self):
		response = self.fetch('/digest', auth_mode='digest',
							  auth_username='foo', auth_password='bar')
		self.assertEqual(response.body, b'ok')

# 需要导入模块: from tornado.curl_httpclient import CurlAsyncHTTPClient [as 别名]
# 或者: from tornado.curl_httpclient.CurlAsyncHTTPClient import fetch [as 别名]
def get_timelapse_movie(local_config, key, group, callback):
    scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
    
    logging.debug('downloading timelapse movie for remote camera %(id)s on %(url)s' % {
            'id': camera_id,
            'url': pretty_camera_url(local_config)})
    
    request = _make_request(scheme, host, port, username, password,
            path + '/picture/%(id)s/timelapse/%(group)s/?key=%(key)s' % {
                'id': camera_id,
                'group': group,
                'key': key},
            timeout=10 * settings.REMOTE_REQUEST_TIMEOUT)

    def on_response(response):
        if response.error:
            logging.error('failed to download timelapse movie for remote camera %(id)s on %(url)s: %(msg)s' % {
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'msg': utils.pretty_http_error(response)})

            return callback(error=utils.pretty_http_error(response))

        callback({
            'data': response.body,
            'content_type': response.headers.get('Content-Type'),
            'content_disposition': response.headers.get('Content-Disposition')
        })

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

# 需要导入模块: from tornado.curl_httpclient import CurlAsyncHTTPClient [as 别名]
# 或者: from tornado.curl_httpclient.CurlAsyncHTTPClient import fetch [as 别名]
 def get_real_ip(self, geolocate=True):
     try:
         self.real_ip = self.request.headers.get(
             'X-Real-Ip',
             self.request.headers.get('X-Forwarded-For', None))
         logging.info(
             "Request from " + str(self.real_ip) + str(self.__class__))
         if geolocate:
             geo_key = "geo_%s" % self.real_ip
             cached_geo = memcache_get(geo_key)
             if cached_geo:
                 logging.info(cached_geo)
             else:
                 def handle_request(responses):
                     geo = load_json(responses.body)
                     memcache_set(geo_key, geo)
                     logging.info(geo)
                 http_client = CurlAsyncHTTPClient()
                 # need to make this a external configuration
                 http_client.fetch("http://freegeoip.net/json/%s" %
                                   self.real_ip,
                                   callback=handle_request,
                                   request_timeout=2,
                                   connect_timeout=2)
     except Exception as e:
         self.error(e)

# 需要导入模块: from tornado.curl_httpclient import CurlAsyncHTTPClient [as 别名]
# 或者: from tornado.curl_httpclient.CurlAsyncHTTPClient import fetch [as 别名]
class jarvis_testcase(tornado.testing.AsyncHTTPTestCase):

    def setUp(self):
        super(jarvis_testcase, self).setUp()
        self.http_client = CurlAsyncHTTPClient(self.io_loop)
        self.headers = {'secret': config.config['secret']}

    def get_app(self):
        self.jarvis = kernel.init(config.config)
        return self.jarvis._application

    @tornado.testing.gen_test
    def http_request(self, request, headers = None):
        command = urllib.quote(request, '')
        uri = command.replace(urllib.quote(' '), '/', 2)
        url = '/api/' + uri

        if headers == None:
            headers = self.headers

        request = HTTPRequest(self.get_url(url), headers=headers)
        response = yield self.http_client.fetch(request, raise_error=False)
        raise tornado.gen.Return(json.loads(response.body))

    @tornado.testing.gen_test
    def raw_http_request(self, url, headers = None):
        if headers == None:
            headers = self.headers

        request = HTTPRequest(self.get_url(url), headers=headers)
        response = yield self.http_client.fetch(request, raise_error=False)
        raise tornado.gen.Return(response)

# 需要导入模块: from tornado.curl_httpclient import CurlAsyncHTTPClient [as 别名]
# 或者: from tornado.curl_httpclient.CurlAsyncHTTPClient import fetch [as 别名]
def get_media_content(local_config, filename, media_type, callback):
    scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
    
    logging.debug('downloading file %(filename)s of remote camera %(id)s on %(url)s' % {
            'filename': filename,
            'id': camera_id,
            'url': pretty_camera_url(local_config)})
    
    path += '/%(media_type)s/%(id)s/download/%(filename)s' % {
            'media_type': media_type,
            'id': camera_id,
            'filename': filename}
    
    # timeout here is 10 times larger than usual - we expect a big delay when fetching the media list
    request = _make_request(scheme, host, port, username, password,
            path, timeout=10 * settings.REMOTE_REQUEST_TIMEOUT)
    
    def on_response(response):
        if response.error:
            logging.error('failed to download file %(filename)s of remote camera %(id)s on %(url)s: %(msg)s' % {
                    'filename': filename,
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'msg': utils.pretty_http_error(response)})
            
            return callback(error=utils.pretty_http_error(response))
        
        return callback(response.body)

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

# 需要导入模块: from tornado.curl_httpclient import CurlAsyncHTTPClient [as 别名]
# 或者: from tornado.curl_httpclient.CurlAsyncHTTPClient import fetch [as 别名]
def check_timelapse_movie(local_config, group, callback):
    scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
    
    logging.debug('checking timelapse movie status for remote camera %(id)s on %(url)s' % {
            'id': camera_id,
            'url': pretty_camera_url(local_config)})
    
    request = _make_request(scheme, host, port, username, password,
            path + '/picture/%(id)s/timelapse/%(group)s/?check=true' % {
                    'id': camera_id,
                    'group': group})
    
    def on_response(response):
        if response.error:
            logging.error('failed to check timelapse movie status for remote camera %(id)s on %(url)s: %(msg)s' % {
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'msg': utils.pretty_http_error(response)})

            return callback(error=utils.pretty_http_error(response))
        
        try:
            response = json.loads(response.body)

        except Exception as e:
            logging.error('failed to decode json answer from %(url)s: %(msg)s' % {
                    'url': pretty_camera_url(local_config),
                    'msg': unicode(e)})

            return callback(error=unicode(e))
        
        callback(response)

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

# 需要导入模块: from tornado.curl_httpclient import CurlAsyncHTTPClient [as 别名]
# 或者: from tornado.curl_httpclient.CurlAsyncHTTPClient import fetch [as 别名]
def test(local_config, data, callback):
    scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
    what = data['what']
    logging.debug('testing %(what)s on remote camera %(id)s, on %(url)s' % {
            'what': what,
            'id': camera_id,
            'url': pretty_camera_url(local_config)})

    data = json.dumps(data)

    request = _make_request(scheme, host, port, username, password,
            path + '/config/%(id)s/test/' % {'id': camera_id},
            method='POST', data=data, content_type='application/json')

    def on_response(response):
        if response.error:
            logging.error('failed to test %(what)s on remote camera %(id)s, on %(url)s: %(msg)s' % {
                    'what': what,
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'msg': utils.pretty_http_error(response)})

            return callback(error=utils.pretty_http_error(response))
        
        callback()

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

# 需要导入模块: from tornado.curl_httpclient import CurlAsyncHTTPClient [as 别名]
# 或者: from tornado.curl_httpclient.CurlAsyncHTTPClient import fetch [as 别名]
def set_preview(local_config, controls, callback):
    scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
    
    logging.debug('setting preview for remote camera %(id)s on %(url)s' % {
            'id': camera_id,
            'url': pretty_camera_url(local_config)})
    
    data = json.dumps(controls)
    
    request = _make_request(scheme, host, port, username, password,
            path + '/config/%(id)s/set_preview/' % {'id': camera_id},
            method='POST', data=data, content_type='application/json')

    def on_response(response):
        if response.error:
            logging.error('failed to set preview for remote camera %(id)s on %(url)s: %(msg)s' % {
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'msg': utils.pretty_http_error(response)})
        
            return callback(error=utils.pretty_http_error(response))
        
        callback()

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

# 需要导入模块: from tornado.curl_httpclient import CurlAsyncHTTPClient [as 别名]
# 或者: from tornado.curl_httpclient.CurlAsyncHTTPClient import fetch [as 别名]
def set_config(local_config, ui_config, callback):
    scheme = local_config.get('@scheme', local_config.get('scheme'))
    host = local_config.get('@host', local_config.get('host')) 
    port = local_config.get('@port', local_config.get('port'))
    username = local_config.get('@username', local_config.get('username'))
    password = local_config.get('@password', local_config.get('password'))
    path = local_config.get('@path', local_config.get('path')) or ''
    camera_id = local_config.get('@remote_camera_id', local_config.get('remote_camera_id'))

    logging.debug('setting config for remote camera %(id)s on %(url)s' % {
            'id': camera_id,
            'url': pretty_camera_url(local_config)})
    
    ui_config = json.dumps(ui_config)
    
    request = _make_request(scheme, host, port, username, password,
            path + '/config/%(id)s/set/' % {'id': camera_id},
            method='POST', data=ui_config, content_type='application/json')
    
    def on_response(response):
        if response.error:
            logging.error('failed to set config for remote camera %(id)s on %(url)s: %(msg)s' % {
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'msg': utils.pretty_http_error(response)})
            
            return callback(error=utils.pretty_http_error(response))
    
        callback()

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

# 需要导入模块: from tornado.curl_httpclient import CurlAsyncHTTPClient [as 别名]
# 或者: from tornado.curl_httpclient.CurlAsyncHTTPClient import fetch [as 别名]
class Scour(object):
    "The base class of the scraper"
    
    def __init__(self, seed_urls=[], url_fetch_timeout=25):
        
        ioloop.IOLoop.instance() #instanceiate IOLoop
        self.client = CurlAsyncHTTPClient()
        
        logging.basicConfig(level=logging.DEBUG, filename="m_scrape.log", filemode='w')
        self.log = logging.getLogger(name="mod_scrape")
        
        self.__extractors = []
        
        #Queue Argumet urls
        for url in seed_urls:
            self.get(url)
            
    def get(self, url):
        self.client.fetch(str(url), self.process_page)
            
    def process_page(self, response):
        print "processing page"
        try:
            soup = bSoup(response.body)
            for x in self.__extractors:             #Run the extractors
                x(self, soup, response)
        except Exception, e:                           #Log any errors
            import traceback
            self.log.error(str(traceback.format_exc(limit=10)))
            self.log.error(str(e))

# 需要导入模块: from tornado.curl_httpclient import CurlAsyncHTTPClient [as 别名]
# 或者: from tornado.curl_httpclient.CurlAsyncHTTPClient import fetch [as 别名]
def exec_action(local_config, action, callback):
    scheme, host, port, username, password, path, camera_id = _remote_params(local_config)
    
    logging.debug('executing action "%(action)s" of remote camera %(id)s on %(url)s' % {
            'action': action,
            'id': camera_id,
            'url': pretty_camera_url(local_config)})

    path += '/action/%(id)s/%(action)s/' % {
            'action': action,
            'id': camera_id}

    request = _make_request(scheme, host, port, username, password, path,
            method='POST', data='{}',
            timeout=settings.REMOTE_REQUEST_TIMEOUT, content_type='application/json')

    def on_response(response):
        if response.error:
            logging.error('failed to execute action "%(action)s" of remote camera %(id)s on %(url)s: %(msg)s' % {
                    'action': action,
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'msg': utils.pretty_http_error(response)})

            return callback(error=utils.pretty_http_error(response))
        
        callback()

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))

# 需要导入模块: from tornado.curl_httpclient import CurlAsyncHTTPClient [as 别名]
# 或者: from tornado.curl_httpclient.CurlAsyncHTTPClient import fetch [as 别名]
    def fetch(self, url, **kwargs):
        # init HTTPRequest
        http_client = CurlAsyncHTTPClient()
        session = HTTPRequest('', follow_redirects=False)
        instance_parameters = copy.deepcopy(self._req_params) # 参数

        self.init_request(session, url, **kwargs)

        while True:

            self.pre_request(session, url=url, **kwargs)
            try:
                fetch_logger.info('{method} {url}'.format(method=session.method, url=session.url))
                response = yield http_client.fetch(session)
                fetch_logger.log_green('{code} {url}'.format(code=response.code, url=session.url))
                break
            except HTTPError as httperr:
                # redirects handler
                if httperr.code > 300 and httperr.code < 400:
                    fetch_logger.warning('{code} {url}'.format(code=httperr.code, url=session.url))
                    self.post_request(session, httperr.response, url, **kwargs)
                    if not kwargs.get('disabled_redirect'):
                        url = httperr.response.headers.get('Location')
                    else:
                        fetch_logger.debug(httperr)
                        raise gen.Return(httperr.response)
                else:
                    fetch_logger.error(httperr)
                    raise httperr


        del instance_parameters
        self.post_request(session, response, url, **kwargs)

        raise gen.Return(response)

# 需要导入模块: from tornado.curl_httpclient import CurlAsyncHTTPClient [as 别名]
# 或者: from tornado.curl_httpclient.CurlAsyncHTTPClient import fetch [as 别名]
 def get(self):
     request_client = HTTPRequest(url = self.request.uri, 
                                  method = self.request.method,
                                  body = self.request.body or None,
                                  headers = self.request.headers)
     http_client = CurlAsyncHTTPClient()
     response = http_client.fetch(request_client, self.response_client)

# 需要导入模块: from tornado.curl_httpclient import CurlAsyncHTTPClient [as 别名]
# 或者: from tornado.curl_httpclient.CurlAsyncHTTPClient import fetch [as 别名]
def make_timelapse_movie(local_config, framerate, interval, group, callback):
    scheme, host, port, username, password, path, camera_id = _remote_params(local_config)

    logging.debug('making timelapse movie for group "%(group)s" of remote camera %(id)s with rate %(framerate)s/%(int)s on %(url)s' % {
            'group': group or 'ungrouped',
            'id': camera_id,
            'framerate': framerate,
            'int': interval,
            'url': pretty_camera_url(local_config)})

    path += '/picture/%(id)s/timelapse/%(group)s/?interval=%(int)s&framerate=%(framerate)s' % {
            'id': camera_id,
            'int': interval,
            'framerate': framerate,
            'group': group}
    
    request = _make_request(scheme, host, port, username, password,
            path, timeout=100 * settings.REMOTE_REQUEST_TIMEOUT)

    def on_response(response):
        if response.error:
            logging.error('failed to make timelapse movie for group "%(group)s" of remote camera %(id)s with rate %(framerate)s/%(int)s on %(url)s: %(msg)s' % {
                    'group': group or 'ungrouped',
                    'id': camera_id,
                    'url': pretty_camera_url(local_config),
                    'int': interval,
                    'framerate': framerate,
                    'msg': utils.pretty_http_error(response)})

            return callback(error=utils.pretty_http_error(response))
        
        try:
            response = json.loads(response.body)

        except Exception as e:
            logging.error('failed to decode json answer from %(url)s: %(msg)s' % {
                    'url': pretty_camera_url(local_config),
                    'msg': unicode(e)})

            return callback(error=unicode(e))
        
        callback(response)

    http_client = CurlAsyncHTTPClient()
    http_client.fetch(request, _callback_wrapper(on_response))