本文整理汇总了Python中fetcher.Fetcher._get_helper方法的典型用法代码示例。如果您正苦于以下问题：Python Fetcher._get_helper方法的具体用法？Python Fetcher._get_helper怎么用？Python Fetcher._get_helper使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类fetcher.Fetcher的用法示例。
在下文中一共展示了Fetcher._get_helper方法的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: _scrape_single_article
# 需要导入模块: from fetcher import Fetcher [as 别名]
# 或者: from fetcher.Fetcher import _get_helper [as 别名]
def _scrape_single_article(self, d):
""" Single article scraper that will be called by a process within a
multiprocessing pool """
try:
helper = Fetcher._get_helper(d.root)
if helper:
self.scrape_article(d.url, helper)
except Exception as e:
logging.warning(
"[{2}] Exception when scraping article at {0}: {1!r}".format(
d.url, e, d.seq
)
)示例2: _scrape_single_root
# 需要导入模块: from fetcher import Fetcher [as 别名]
# 或者: from fetcher.Fetcher import _get_helper [as 别名]
def _scrape_single_root(self, r):
""" Single root scraper that will be called by a process within a
multiprocessing pool """
if r.domain.endswith(".local"):
# We do not scrape .local roots
return
try:
logging.info("Scraping root of {0} at {1}...".format(r.description, r.url))
# Process a single top-level domain and root URL,
# parsing child URLs that have not been seen before
helper = Fetcher._get_helper(r)
if helper:
self.scrape_root(r, helper)
except Exception as e:
logging.warning(
"Exception when scraping root at {0}: {1!r}".format(r.url, e)
)示例3: _parse_single_article
# 需要导入模块: from fetcher import Fetcher [as 别名]
# 或者: from fetcher.Fetcher import _get_helper [as 别名]
def _parse_single_article(self, d):
""" Single article parser that will be called by a process within a
multiprocessing pool """
try:
helper = Fetcher._get_helper(d.root)
if helper:
self.parse_article(d.seq, d.url, helper)
except KeyboardInterrupt:
logging.info("KeyboardInterrupt in _parse_single_article()")
sys.exit(1)
except MemoryError:
# Nothing to do but give up on this process
sys.exit(1)
except Exception as e:
logging.warning(
"[{2}] Exception when parsing article at {0}: {1!r}".format(
d.url, e, d.seq
)
)
# traceback.print_exc()
# raise
return True