本文整理汇总了Python中scrapy.FormRequest.meta["page_base_url"]方法的典型用法代码示例。如果您正苦于以下问题：Python FormRequest.meta["page_base_url"]方法的具体用法？Python FormRequest.meta["page_base_url"]怎么用？Python FormRequest.meta["page_base_url"]使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类scrapy.FormRequest的用法示例。
在下文中一共展示了FormRequest.meta["page_base_url"]方法的2个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: parse_middle_category
# 需要导入模块: from scrapy import FormRequest [as 别名]
# 或者: from scrapy.FormRequest import meta["page_base_url"] [as 别名]
def parse_middle_category(self, response):
report_types = response.xpath(u"//li[contains(text(),'报告')]")
for report_type in report_types:
mid_category_url = urljoin(self.base_url, report_type.xpath(u"./preceding-sibling::span[1]/a/@href").extract()[0])
request = FormRequest(mid_category_url, callback=self.parse_page, dont_filter=True)
request.meta["large_category_name"] = response.meta["large_category_name"]
request.meta["mid_category_name"] = response.meta["mid_category_name"]
request.meta["report_type"] = clean_text(report_type.xpath("./text()").extract()[0].strip())
request.meta["page_base_url"] = mid_category_url
yield request示例2: parse_page
# 需要导入模块: from scrapy import FormRequest [as 别名]
# 或者: from scrapy.FormRequest import meta["page_base_url"] [as 别名]
def parse_page(self, response):
request_list = self._parse_list(response)
for r in request_list:
yield r
next_page = response.xpath(u"//*[@id='AspNetPager1']/a[text()=\"下一页\"]/@href")
if len(next_page) > 0:
next_page_url = urljoin(self.base_url, next_page.extract()[0])
if not next_page_url.startswith(response.meta["page_base_url"]):
if next_page_url.endswith("html"):
next_page_url = response.meta["page_base_url"] + next_page_url[next_page_url.rindex("/") + 1:len(next_page_url)]
request = FormRequest(next_page_url, callback=self.parse_page, dont_filter=True)
request.meta["large_category_name"] = response.meta["large_category_name"]
request.meta["mid_category_name"] = response.meta["mid_category_name"]
request.meta["report_type"] = response.meta["report_type"]
request.meta["page_base_url"] = response.meta["page_base_url"]
yield request