本文整理汇总了Python中scrapy.FormRequest.from_response方法的典型用法代码示例。如果您正苦于以下问题：Python FormRequest.from_response方法的具体用法？Python FormRequest.from_response怎么用？Python FormRequest.from_response使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类scrapy.FormRequest的用法示例。
在下文中一共展示了FormRequest.from_response方法的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: parse
# 需要导入模块: from scrapy import FormRequest [as 别名]
# 或者: from scrapy.FormRequest import from_response [as 别名]
def parse(self, response) :
# test_urls = [
# "http://ntiaoji.kaoyan.com/tjadm/1.html",
# "http://ntiaoji.kaoyan.com/tjadm/2.html",
# "http://ntiaoji.kaoyan.com/tjadm/3.html",
# "http://ntiaoji.kaoyan.com/tjadm/4.html",
# "http://ntiaoji.kaoyan.com/tjadm/5.html",
# "http://ntiaoji.kaoyan.com/tjadm/6.html",
# "http://ntiaoji.kaoyan.com/tjadm/7.html"
# ]
#
# for url in test_urls :
# print url
# time.sleep(2)
# self.headers['Referer'] = url
# yield FormRequest.from_response(response,
# headers = self.headers,
# formdata = {
# 'username' : 'kytj1',
# 'password' : '6ujBJ4XQyLeGmJmB'
# },
# callback = self.download_page,
# dont_filter = True
# )
return FormRequest.from_response(response,
headers = self.headers,
formdata = {
'username' : 'kytj1',
'password' : '6ujBJ4XQyLeGmJmB'
},
callback = self.after_login,
dont_filter = True
)示例2: parse
# 需要导入模块: from scrapy import FormRequest [as 别名]
# 或者: from scrapy.FormRequest import from_response [as 别名]
def parse(self, response):
"""
这是默认的回调方法，得到response后：
1. 如果需要登录，则先通过FormRequest登录论坛；
2. 如果不需要登录，通过Request继续请求；
:param response:
:return:
"""
# 需要登录，使用FormRequest.from_response模拟登录
if 'id="lsform"' in response.body:
logging.info('in parse, need to login, url: {0}'.format(response.url))
form_data = {'handlekey': 'ls', 'quickforward': 'yes', 'username': 'daniell123', 'password': 'admin123'}
request = FormRequest.from_response(response=response,
headers=self.headers,
formxpath='//form[contains(@id, "lsform")]',
formdata=form_data,
callback=self.parse_list
)
else:
logging.info('in parse, NOT need to login, url: {0}'.format(response.url))
request = Request(url=response.url,
headers=self.headers,
callback=self.parse_list,
)
yield request示例3: _request_next_page
# 需要导入模块: from scrapy import FormRequest [as 别名]
# 或者: from scrapy.FormRequest import from_response [as 别名]
def _request_next_page(self, response, date_str, callback):
current_page = int(response.meta['current_page'])
total_string = response.css('#LblTotal').xpath('./text()').extract_first(default='')
total = re.search(r'(\d+)', total_string)
if total:
# Deal with the next page.
total = total.group(1)
number_of_pages = self._get_number_of_pages(int(total))
if current_page < number_of_pages:
current_page += 1
formdata = {
'TxtFecha': date_str,
'BtnBuscar': 'Buscar',
'LwVisitasCR$DpVisitasCR$ctl02$ctl00.x': '1',
'LwVisitasCR$DpVisitasCR$ctl02$ctl00.y': '1'
}
request = FormRequest.from_response(response,
formdata=formdata,
dont_click=True,
dont_filter=True,
callback=callback,
)
request.meta['date'] = date_str
request.meta['current_page'] = current_page
return request示例4: parse
# 需要导入模块: from scrapy import FormRequest [as 别名]
# 或者: from scrapy.FormRequest import from_response [as 别名]
def parse(self, response):
form_data = {'username': '[email protected]', 'password': '123456', 'remember_me': '1'}
return FormRequest.from_response(response,
headers=self.headers,
formxpath='//form[@class="form-login"]',
formdata=form_data,
callback=self.after_login,
)示例5: parse
# 需要导入模块: from scrapy import FormRequest [as 别名]
# 或者: from scrapy.FormRequest import from_response [as 别名]
def parse(self, response):
yield FormRequest.from_response(
response,
formname='aspnetForm',
formdata={'Skin$body$FundingSourceChoices$0': '1',
'Skin$body$FundingSourceChoices$1': '0'},
meta={'curr_listing_page': 1, 'flag': False},
callback=self.after_login)示例6: parse
# 需要导入模块: from scrapy import FormRequest [as 别名]
# 或者: from scrapy.FormRequest import from_response [as 别名]
def parse(self, response):
login_form = {
'login': self.username,
'password': self.password,
}
return FormRequest.from_response(
response,
formdata=login_form,
callback=self.after_login
)示例7: parse
# 需要导入模块: from scrapy import FormRequest [as 别名]
# 或者: from scrapy.FormRequest import from_response [as 别名]
def parse(self, response) :
return FormRequest.from_response(response,
headers = self.headers,
formdata = {
'username' : 'kytj1',
'password' : '6ujBJ4XQyLeGmJmB'
},
callback = self.after_login,
dont_filter = True
)示例8: parse
# 需要导入模块: from scrapy import FormRequest [as 别名]
# 或者: from scrapy.FormRequest import from_response [as 别名]
def parse(self, response):
yield FormRequest.from_response(response,
formdata={
'tanggal': '20160817#Rabu, 17 Agustus 2016',
'origination': 'KAC#KIARACONDONG',
'destination': 'MN#MADIUN',
'adult': '1',
'infant': '0'
},
callback=self.parseInfo)示例9: login
# 需要导入模块: from scrapy import FormRequest [as 别名]
# 或者: from scrapy.FormRequest import from_response [as 别名]
def login(self,response):
# login = requests.post(response.url,
# headers = self.headers,
# data={
# 'source':'None',
# 'redir':'https://www.douban.com/people/60012975/',
# 'form_email':'[email protected]',
# 'form_password':'123456789',
#
# 'remember':'on',
# 'login':u'登录'
# })
hxs = Selector(response)
if hxs.xpath('//*[@name="captcha-id"]/@value').extract():
captchaID = hxs.xpath('//*[@name="captcha-id"]/@value').extract()[0]
captchAdd = hxs.xpath('//*[@id="captcha_image"]/@src').extract()[0]
urllib.urlretrieve(captchAdd,'captcha.jpg')
captch = raw_input('please input the captcha:')
yield FormRequest.from_response(response,
meta =response.meta,
# headers = self.headers,
formdata={'source':'None',
'redir':'https://www.douban.com/people/unlucky_strike/',
'form_email':'[email protected]',
'form_password':'123456789',
'captcha-solution':captch,
'captcha-id':captchaID,
'remember':'on',
'login':u'登录'},
callback=self.parse)
else:
yield FormRequest.from_response(response,
meta ={'cookiejar':response.meta['cookiejar']},
# headers = self.headers,
formdata={'source':'None',
'redir':'https://www.douban.com/people/unlucky_strike/',
'form_email':'[email protected]',
'form_password':'123456789',
'remember':'on',
'login':u'登录'},
callback=self.parse)示例10: parse
# 需要导入模块: from scrapy import FormRequest [as 别名]
# 或者: from scrapy.FormRequest import from_response [as 别名]
def parse(self, response):
'''Parse login page'''
return FormRequest.from_response(
response,
formxpath='//form[contains(@action, "login")]',
formdata={
'email': self.username,
'pass': self.password,
},
callback=self.parse_home,
)示例11: _get_page_request
# 需要导入模块: from scrapy import FormRequest [as 别名]
# 或者: from scrapy.FormRequest import from_response [as 别名]
def _get_page_request(self, response, page, date):
request = FormRequest.from_response(
response,
formdata={"txtDesde": date, "__EVENTTARGET": "gvwConsulta", "__EVENTARGUMENT": "Page${}".format(page)},
dont_filter=True,
callback=self.parse,
)
request.meta["date"] = date
return request示例12: currency_form
# 需要导入模块: from scrapy import FormRequest [as 别名]
# 或者: from scrapy.FormRequest import from_response [as 别名]
def currency_form(self, response):
"""
Currency form viewed and change to USD posted.
"""
self.log('currency_form', level=logging.INFO)
formdata = {
'ddlCountry1': 'United States',
'ddlCurrency': '503329C6-40CB-47E6-91D1-9F11AF63F706'
}
return FormRequest.from_response(response,
formdata=formdata,
callback=self.currency_changed)示例13: parse_initial_request
# 需要导入模块: from scrapy import FormRequest [as 别名]
# 或者: from scrapy.FormRequest import from_response [as 别名]
def parse_initial_request(self, response):
date = response.meta["date"]
request = FormRequest.from_response(
response,
formdata={"txtDesde": date, "btnBuscar.x": "1", "btnBuscar.y": "1"},
dont_filter=True,
callback=self.parse_page,
)
request.meta["date"] = date
yield request示例14: parse_home
# 需要导入模块: from scrapy import FormRequest [as 别名]
# 或者: from scrapy.FormRequest import from_response [as 别名]
def parse_home(self, response):
'''Parse user news feed page'''
if response.css('#approvals_code'):
# Handle 'Approvals Code' checkpoint (ask user to enter code).
if not self.code:
# Show facebook messages via logs
# and request user for approval code.
message = response.css('._50f4::text').extract()[0]
self.log(process_string(message))
message = response.css('._3-8y._50f4').xpath('string()').extract()[0]
self.log(process_string(message))
self.code = input('Enter the code: ')
self.code = str(self.code)
if not (self.code and self.code.isdigit()):
self.log('Bad approvals code detected.')
return
return FormRequest.from_response(
response,
formdata={'approvals_code': self.code},
callback=self.parse_home,
)
elif response.css('input#u_0_1'):
# Handle 'Save Browser' checkpoint.
return FormRequest.from_response(
response,
formdata={'name_action_selected': 'dont_save'},
callback=self.parse_home,
dont_filter=True,
)
elif response.css('button#checkpointSubmitButton'):
# Handle `Someone tried to log into your account` warning.
return FormRequest.from_response(
response, callback=self.parse_home, dont_filter=True,)
# Else go to the user profile.
href = response.css('a[title="Profile"]::attr(href)').extract()[0]
return Request(
response.urljoin(href),
callback=self.parse_profile,
)示例15: parse
# 需要导入模块: from scrapy import FormRequest [as 别名]
# 或者: from scrapy.FormRequest import from_response [as 别名]
def parse(self, response):
"""
Overwrites Spiders parse method. Fill in log in details in log in form and submit.
:return:
"""
print('custom settings:')
print(self._settings)
return FormRequest.from_response(
response,
formxpath='//div[contains(concat(" ", normalize-space(@class), " "), " main-container ")]/descendant::form',
formdata={'EmailOrUsername': self._settings['username'], 'Password': self._settings['password']},
callback=self.go_to_search_site
)