本文整理匯總了Python中fetcher.Fetcher._get_helper方法的典型用法代碼示例。如果您正苦於以下問題：Python Fetcher._get_helper方法的具體用法？Python Fetcher._get_helper怎麽用？Python Fetcher._get_helper使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類fetcher.Fetcher的用法示例。
在下文中一共展示了Fetcher._get_helper方法的3個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: _scrape_single_article
# 需要導入模塊: from fetcher import Fetcher [as 別名]
# 或者: from fetcher.Fetcher import _get_helper [as 別名]
def _scrape_single_article(self, d):
""" Single article scraper that will be called by a process within a
multiprocessing pool """
try:
helper = Fetcher._get_helper(d.root)
if helper:
self.scrape_article(d.url, helper)
except Exception as e:
logging.warning(
"[{2}] Exception when scraping article at {0}: {1!r}".format(
d.url, e, d.seq
)
)示例2: _scrape_single_root
# 需要導入模塊: from fetcher import Fetcher [as 別名]
# 或者: from fetcher.Fetcher import _get_helper [as 別名]
def _scrape_single_root(self, r):
""" Single root scraper that will be called by a process within a
multiprocessing pool """
if r.domain.endswith(".local"):
# We do not scrape .local roots
return
try:
logging.info("Scraping root of {0} at {1}...".format(r.description, r.url))
# Process a single top-level domain and root URL,
# parsing child URLs that have not been seen before
helper = Fetcher._get_helper(r)
if helper:
self.scrape_root(r, helper)
except Exception as e:
logging.warning(
"Exception when scraping root at {0}: {1!r}".format(r.url, e)
)示例3: _parse_single_article
# 需要導入模塊: from fetcher import Fetcher [as 別名]
# 或者: from fetcher.Fetcher import _get_helper [as 別名]
def _parse_single_article(self, d):
""" Single article parser that will be called by a process within a
multiprocessing pool """
try:
helper = Fetcher._get_helper(d.root)
if helper:
self.parse_article(d.seq, d.url, helper)
except KeyboardInterrupt:
logging.info("KeyboardInterrupt in _parse_single_article()")
sys.exit(1)
except MemoryError:
# Nothing to do but give up on this process
sys.exit(1)
except Exception as e:
logging.warning(
"[{2}] Exception when parsing article at {0}: {1!r}".format(
d.url, e, d.seq
)
)
# traceback.print_exc()
# raise
return True