本文整理匯總了Python中scrapy.item.Item方法的典型用法代碼示例。如果您正苦於以下問題:Python item.Item方法的具體用法?Python item.Item怎麽用?Python item.Item使用的例子?那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類scrapy.item
的用法示例。
在下文中一共展示了item.Item方法的4個代碼示例,這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚,您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: main
# 需要導入模塊: from scrapy import item [as 別名]
# 或者: from scrapy.item import Item [as 別名]
def main():
"""Main routine for the execution of the Spider"""
# set up signal to catch items scraped
def catch_item(sender, item, **kwargs):
print("Item extracted:", item)
dispatcher.connect(catch_item, signal=signals.item_passed)
settings = Settings()
settings.set("USER_AGENT", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36")
settings.set("LOG_ENABLED",False)
# setup crawler
from scrapy.crawler import CrawlerProcess
crawler = CrawlerProcess(settings)
# define the spider for the crawler
crawler.crawl(EuropythonSpyder())
# start scrapy
print("STARTING ENGINE")
crawler.start() #iniciar el crawler llamando al spider definido
print("ENGINE STOPPED")
開發者ID:PacktPublishing,項目名稱:Learning-Python-Networking-Second-Edition,代碼行數:25,代碼來源:EuropythonSpyder.py
示例2: binary_check
# 需要導入模塊: from scrapy import item [as 別名]
# 或者: from scrapy.item import Item [as 別名]
def binary_check(fx_obj, cb_obj, encoding):
if isinstance(cb_obj, (dict, Item)):
fx_obj = {
key: binary_check(value, cb_obj[key], encoding)
for key, value in fx_obj.items()
}
if isinstance(cb_obj, list):
fx_obj = [
binary_check(fxitem, cbitem, encoding)
for fxitem, cbitem in zip(fx_obj, cb_obj)
]
if isinstance(cb_obj, Request):
headers = {}
for key, value in fx_obj['headers'].items():
key = to_bytes(key, encoding)
headers[key] = [to_bytes(v, encoding) for v in value]
fx_obj['headers'] = headers
fx_obj['body'] = to_bytes(fx_obj['body'], encoding)
if isinstance(cb_obj, six.binary_type):
fx_obj = fx_obj.encode(encoding)
return fx_obj
示例3: another_process_item
# 需要導入模塊: from scrapy import item [as 別名]
# 或者: from scrapy.item import Item [as 別名]
def another_process_item(self, result, item, info):
"""
custom process_item func,so it will manage the Request result.
"""
assert isinstance(result, (Item, Request)), \
"WoaiduBookFile pipeline' item_completed must return Item or Request, got %s" % \
(type(result))
if isinstance(result, Item):
return result
elif isinstance(result, Request):
dlist = [self._process_request(r, info) for r in arg_to_iter(result)]
dfd = DeferredList(dlist, consumeErrors=1)
dfd.addCallback(self.item_completed, item, info)
# XXX:This will cause one item maybe return many times,it depends on how many
# times the download url failed.But it doesn't matter.Because when raise errors,
# the items are no longer processed by further pipeline components.And when all
# url download failed we can drop that item which book_file or book_file_url are
# empty.
return dfd.addCallback(self.another_process_item, item, info)
else:
raise NofilesDrop
示例4: _get_item_field_attr
# 需要導入模塊: from scrapy import item [as 別名]
# 或者: from scrapy.item import Item [as 別名]
def _get_item_field_attr(self, field_name, key, default=None):
if isinstance(self.item, Item):
value = self.item.fields[field_name].get(key, default)
else:
value = default
return value