本文整理汇总了Python中scrapy.spiders.Rule方法的典型用法代码示例。如果您正苦于以下问题：Python spiders.Rule方法的具体用法？Python spiders.Rule怎么用？Python spiders.Rule使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类scrapy.spiders的用法示例。
在下文中一共展示了spiders.Rule方法的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: parse_body
# 需要导入模块: from scrapy import spiders [as 别名]
# 或者: from scrapy.spiders import Rule [as 别名]
def parse_body(self, response):
global crawl_counter
try:
crawl_counter += 1
if crawl_counter > max_crawl_limit:
os._exit(0)
matches = rules.match(data=response.body)
if matches:
for match in matches:
print "[+] URL:", response.request.url, "Matched YARA Rule:", match
if debug:
print "[*] Matched body response:", response.body.decode("utf-8")
txt = "URL: " + response.request.url + " Matched YARA Rule: " + str(match)
log(txt)
except Exception as e:
# if debug:
print str(e) 示例2: request_index
# 需要导入模块: from scrapy import spiders [as 别名]
# 或者: from scrapy.spiders import Rule [as 别名]
def request_index(self, response):
categories = list(set(response.css('#topMenuItem a::attr("href")').re('/([^\/]+)/$')))
if self.category is not None:
if self.category in categories:
categories = [self.category]
else:
raise ValueError('invalid category slug. available slugs: %s' % ", ".join(categories))
date_processing = self.start_date
while date_processing <= self.end_date:
for category in categories:
# redifining the rule again according to the specific date url
SamakalSpider.rules = (Rule(LinkExtractor(allow=('/' + date_processing.strftime('%Y/%m/%d') + '/\d+$',),
restrict_xpaths=('//div[@class="main-body"]')),
callback="parse_content", follow=True),)
super(SamakalSpider, self)._compile_rules()
# http://bangla.samakal.net/-education/2016/06/01
url = 'http://bangla.samakal.net/{0}/{1}'.format(
category,
date_processing.strftime('%Y/%m/%d')
)
yield self.make_requests_from_url(url)
date_processing += datetime.timedelta(days=1) 示例3: __init__
# 需要导入模块: from scrapy import spiders [as 别名]
# 或者: from scrapy.spiders import Rule [as 别名]
def __init__(self, domains, urls, *args, **kwargs):
"""Constructor for SiteSpider.
Parameters
----------
domains : list
A list of domains for the site.
urls : list
A list of sitemap URLS of the site.
href_xpaths : list
A list of XPATH expression indicating the ancestors of `<a>`
element.
url_regex : string
URL pattern regular expression.
If you use this spider to store item into database, additional
keywords are required:
platform_id : int
The id of a platform instance.
session : object
An instance of SQLAlchemy session.
"""
self.session = kwargs.pop('session', None)
self.platform_id = kwargs.pop('platform_id', None)
self.url_regex = kwargs.pop('url_regex', None)
self.href_xpaths = kwargs.pop('href_xpaths', ())
self.start_urls = urls
self.allowed_domains = domains
self.rules = (Rule(
LinkExtractor(
allow_domains=self.allowed_domains,
restrict_xpaths=self.href_xpaths,
unique=True),
callback="parse_item",
follow=True),)
super(SiteSpider, self).__init__(*args, **kwargs)