本文整理汇总了Python中pyspider.fetcher.tornado_fetcher.Fetcher.run方法的典型用法代码示例。如果您正苦于以下问题：Python Fetcher.run方法的具体用法？Python Fetcher.run怎么用？Python Fetcher.run使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类pyspider.fetcher.tornado_fetcher.Fetcher的用法示例。
在下文中一共展示了Fetcher.run方法的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: run_fetcher
# 需要导入模块: from pyspider.fetcher.tornado_fetcher import Fetcher [as 别名]
# 或者: from pyspider.fetcher.tornado_fetcher.Fetcher import run [as 别名]
def run_fetcher(g=g):
from pyspider.fetcher.tornado_fetcher import Fetcher
fetcher = Fetcher(inqueue=g.scheduler2fetcher, outqueue=g.fetcher2processor)
fetcher.phantomjs_proxy = g.phantomjs_proxy
run_in_thread(fetcher.xmlrpc_run, port=g.fetcher_xmlrpc_port, bind=g.webui_host)
fetcher.run()示例2: fetcher
# 需要导入模块: from pyspider.fetcher.tornado_fetcher import Fetcher [as 别名]
# 或者: from pyspider.fetcher.tornado_fetcher.Fetcher import run [as 别名]
def fetcher(ctx, xmlrpc, xmlrpc_host, xmlrpc_port):
g = ctx.obj
from pyspider.fetcher.tornado_fetcher import Fetcher
fetcher = Fetcher(inqueue=g.scheduler2fetcher, outqueue=g.fetcher2processor)
fetcher.phantomjs_proxy = g.phantomjs_proxy
g.instances.append(fetcher)
if xmlrpc:
run_in_thread(fetcher.xmlrpc_run, port=xmlrpc_port, bind=xmlrpc_host)
fetcher.run()示例3: fetcher
# 需要导入模块: from pyspider.fetcher.tornado_fetcher import Fetcher [as 别名]
# 或者: from pyspider.fetcher.tornado_fetcher.Fetcher import run [as 别名]
def fetcher(ctx, xmlrpc, xmlrpc_host, xmlrpc_port, poolsize, proxy, user_agent, timeout, Fetcher=Fetcher):
g = ctx.obj
fetcher = Fetcher(inqueue=g.scheduler2fetcher, outqueue=g.fetcher2processor,
poolsize=poolsize, proxy=proxy)
fetcher.phantomjs_proxy = g.phantomjs_proxy
if user_agent:
fetcher.user_agent = user_agent
if timeout:
fetcher.default_options = dict(fetcher.default_options)
fetcher.default_options['timeout'] = timeout
g.instances.append(fetcher)
if g.get('testing_mode'):
return fetcher
if xmlrpc:
run_in_thread(fetcher.xmlrpc_run, port=xmlrpc_port, bind=xmlrpc_host)
fetcher.run()示例4: run_fetcher
# 需要导入模块: from pyspider.fetcher.tornado_fetcher import Fetcher [as 别名]
# 或者: from pyspider.fetcher.tornado_fetcher.Fetcher import run [as 别名]
def run_fetcher(g=g):
from pyspider.fetcher.tornado_fetcher import Fetcher
fetcher = Fetcher(inqueue=g.scheduler2fetcher, outqueue=g.fetcher2processor)
g.fetcher = fetcher
run_in_thread(fetcher.xmlrpc_run)
fetcher.run()