本文整理汇总了Python中slybot.generic_form.GenericForm.set_values_url_field方法的典型用法代码示例。如果您正苦于以下问题：Python GenericForm.set_values_url_field方法的具体用法？Python GenericForm.set_values_url_field怎么用？Python GenericForm.set_values_url_field使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类slybot.generic_form.GenericForm的用法示例。
在下文中一共展示了GenericForm.set_values_url_field方法的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: IblSpider
# 需要导入模块: from slybot.generic_form import GenericForm [as 别名]
# 或者: from slybot.generic_form.GenericForm import set_values_url_field [as 别名]
#.........这里部分代码省略.........
def parse_login_page(self, response):
username = response.request.meta["username"]
password = response.request.meta["password"]
args, url, method = fill_login_form(response.url, response.body,
username, password)
return FormRequest(url, method=method, formdata=args,
callback=self.after_login, dont_filter=True)
def after_login(self, response):
for result in self.parse(response):
yield result
for req in self._start_requests:
yield req
def get_generic_form_start_request(self, form_descriptor):
file_fields = list(self.generic_form.get_url_field(form_descriptor))
if file_fields:
(field_index, field_descriptor) = file_fields.pop(0)
form_descriptor['field_index'] = field_index
return FormRequest(self.generic_form.get_value(field_descriptor),
meta=form_descriptor,
callback=self.parse_field_url_page,
dont_filter=True)
else:
return Request(url=form_descriptor.pop("form_url"),
meta=form_descriptor, callback=self.parse_form_page,
dont_filter=True)
def parse_field_url_page(self, response):
form_descriptor = response.request.meta
field_index = form_descriptor['field_index']
field_descriptor = form_descriptor['fields'][field_index]
self.generic_form.set_values_url_field(field_descriptor, response.body)
yield self.get_generic_form_start_request(form_descriptor)
def parse_form_page(self, response):
fill_form = self.generic_form.fill_generic_form
try:
for (args, url, method) in fill_form(response.url, response.body,
response.request.meta):
yield FormRequest(url, method=method, formdata=args,
callback=self.after_form_page,
dont_filter=True)
except Exception as e:
self.logger.warning(str(e))
for req in self._start_requests:
yield req
def after_form_page(self, response):
for result in self.parse(response):
yield result
def _get_allowed_domains(self, templates):
urls = [x['url'] for x in templates]
urls += [x.url for x in self._start_requests]
return [x[1] for x in iter_unique_scheme_hostname(urls)]
def start_requests(self):
start_requests = []
if self.login_requests:
start_requests = self.login_requests
elif self.form_requests:
start_requests = self.form_requests
else:
start_requests = self._start_requests示例2: IblSpider
# 需要导入模块: from slybot.generic_form import GenericForm [as 别名]
# 或者: from slybot.generic_form.GenericForm import set_values_url_field [as 别名]
#.........这里部分代码省略.........
_links_item_descriptor = create_slybot_item_descriptor({'fields': {}})
self._links_ibl_extractor = InstanceBasedLearningExtractor([(t, _links_item_descriptor) for t in _links_pages]) \
if _links_pages else None
self._ipages = [page for _, page, _ in self._item_template_pages]
self.html_link_extractor = HtmlLinkExtractor()
self.rss_link_extractor = RssLinkExtractor()
self.build_url_filter(spec)
self.itemcls_info = {}
for itemclass_name, triplets in itertools.groupby(self._item_template_pages, operator.itemgetter(0)):
page_extractors_pairs = map(operator.itemgetter(1, 2), triplets)
schema = item_schemas[itemclass_name]
item_cls = SlybotItem.create_iblitem_class(schema)
page_descriptor_pairs = []
for page, template_extractors in page_extractors_pairs:
item_descriptor = create_slybot_item_descriptor(schema)
apply_extractors(item_descriptor, template_extractors, all_extractors)
page_descriptor_pairs.append((page, item_descriptor))
extractor = InstanceBasedLearningExtractor(page_descriptor_pairs)
self.itemcls_info[itemclass_name] = {
'class': item_cls,
'descriptor': item_descriptor,
'extractor': extractor,
}
self.login_requests = []
self.form_requests = []
self._start_requests = []
self.generic_form = GenericForm(**kw)
self._create_init_requests(spec.get("init_requests", []))
self._process_start_urls(spec)
self.allowed_domains = spec.get('allowed_domains',
self._get_allowed_domains(self._ipages))
if not self.allowed_domains:
self.allowed_domains = None
def _process_start_urls(self, spec):
self.start_urls = self.start_urls or spec.get('start_urls')
if isinstance(self.start_urls, basestring):
self.start_urls = self.start_urls.splitlines()
for url in self.start_urls:
self._start_requests.append(Request(url, callback=self.parse, dont_filter=True))
def _create_init_requests(self, spec):
for rdata in spec:
if rdata["type"] == "login":
request = Request(url=rdata.pop("loginurl"), meta=rdata,
callback=self.parse_login_page, dont_filter=True)
self.login_requests.append(request)
elif rdata["type"] == "form":
self.form_requests.append(self.get_generic_form_start_request(rdata))
elif rdata["type"] == "start":
self._start_requests.append(self._create_start_request_from_specs(rdata))
def parse_login_page(self, response):
username = response.request.meta["username"]
password = response.request.meta["password"]
args, url, method = fill_login_form(response.url, response.body, username, password)
return FormRequest(url, method=method, formdata=args, callback=self.after_login, dont_filter=True)
def after_login(self, response):
for result in self.parse(response):
yield result
for req in self._start_requests:
yield req
def get_generic_form_start_request(self, form_descriptor):
file_fields = list(self.generic_form.get_url_field(form_descriptor))
if file_fields:
(field_index, field_descriptor) = file_fields.pop(0)
form_descriptor['field_index'] = field_index
return FormRequest(self.generic_form.get_value(field_descriptor), meta=form_descriptor,
callback=self.parse_field_url_page, dont_filter=True)
else:
return Request(url=form_descriptor.pop("form_url"), meta=form_descriptor,
callback=self.parse_form_page, dont_filter=True)
def parse_field_url_page(self, response):
form_descriptor = response.request.meta
field_index = form_descriptor['field_index']
field_descriptor = form_descriptor['fields'][field_index]
self.generic_form.set_values_url_field(field_descriptor, response.body)
yield self.get_generic_form_start_request(form_descriptor)
def parse_form_page(self, response):
try:
for (args, url, method) in self.generic_form.fill_generic_form(response.url,
response.body,
response.request.meta):
yield FormRequest(url, method=method, formdata=args,
callback=self.after_form_page, dont_filter=True)
except Exception, e:
self.log(str(e), log.WARNING)
for req in self._start_requests:
yield req示例3: IblSpider
# 需要导入模块: from slybot.generic_form import GenericForm [as 别名]
# 或者: from slybot.generic_form.GenericForm import set_values_url_field [as 别名]
class IblSpider(Spider):
def __init__(self, name, spec, item_schemas, all_extractors, settings=None,
**kw):
super(IblSpider, self).__init__(name, **kw)
spec = deepcopy(spec)
for key, val in kw.items():
if isinstance(val, basestring) and key in STRING_KEYS:
val = val.splitlines()
spec[key] = val
self._item_template_pages = sorted(
((t['scrapes'], t) for t in spec['templates']
if t.get('page_type', 'item') == 'item'), key=itemgetter(0))
self._templates = [templ for _, templ in self._item_template_pages]
self.plugins = IndexedDict()
for plugin_class, plugin_name in zip(load_plugins(settings),
load_plugin_names(settings)):
instance = plugin_class()
instance.setup_bot(settings, spec, item_schemas, all_extractors)
self.plugins[plugin_name] = instance
self.login_requests = []
self.form_requests = []
self._start_requests = []
self.generic_form = GenericForm(**kw)
self._create_init_requests(spec.get("init_requests", []))
self._process_start_urls(spec)
self.allowed_domains = spec.get(
'allowed_domains',
self._get_allowed_domains(self._templates)
)
if not self.allowed_domains:
self.allowed_domains = None
def _process_start_urls(self, spec):
self.start_urls = spec.get('start_urls')
for url in self.start_urls:
self._start_requests.append(Request(url, callback=self.parse,
dont_filter=True))
def _create_init_requests(self, spec):
for rdata in spec:
if rdata["type"] == "login":
request = Request(url=rdata.pop("loginurl"), meta=rdata,
callback=self.parse_login_page,
dont_filter=True)
self.login_requests.append(request)
elif rdata["type"] == "form":
self.form_requests.append(
self.get_generic_form_start_request(rdata)
)
elif rdata["type"] == "start":
self._start_requests.append(
self._create_start_request_from_specs(rdata)
)
def parse_login_page(self, response):
username = response.request.meta["username"]
password = response.request.meta["password"]
args, url, method = fill_login_form(response.url, response.body,
username, password)
return FormRequest(url, method=method, formdata=args,
callback=self.after_login, dont_filter=True)
def after_login(self, response):
for result in self.parse(response):
yield result
for req in self._start_requests:
yield req
def get_generic_form_start_request(self, form_descriptor):
file_fields = list(self.generic_form.get_url_field(form_descriptor))
if file_fields:
(field_index, field_descriptor) = file_fields.pop(0)
form_descriptor['field_index'] = field_index
return FormRequest(self.generic_form.get_value(field_descriptor),
meta=form_descriptor,
callback=self.parse_field_url_page,
dont_filter=True)
else:
return Request(url=form_descriptor.pop("form_url"),
meta=form_descriptor, callback=self.parse_form_page,
dont_filter=True)
def parse_field_url_page(self, response):
form_descriptor = response.request.meta
field_index = form_descriptor['field_index']
field_descriptor = form_descriptor['fields'][field_index]
self.generic_form.set_values_url_field(field_descriptor, response.body)
yield self.get_generic_form_start_request(form_descriptor)
def parse_form_page(self, response):
fill_form = self.generic_form.fill_generic_form
try:
for (args, url, method) in fill_form(response.url, response.body,
response.request.meta):
yield FormRequest(url, method=method, formdata=args,
#.........这里部分代码省略.........
示例4: IblSpider
# 需要导入模块: from slybot.generic_form import GenericForm [as 别名]
# 或者: from slybot.generic_form.GenericForm import set_values_url_field [as 别名]
#.........这里部分代码省略.........
def parse_login_page(self, response):
username = response.request.meta["username"]
password = response.request.meta["password"]
args, url, method = fill_login_form(response.url, response.body,
username, password)
return FormRequest(url, method=method, formdata=args,
callback=self.after_login, dont_filter=True)
def after_login(self, response):
for result in self.parse(response):
yield result
for req in self._start_requests:
yield req
def get_generic_form_start_request(self, form_descriptor):
file_fields = list(self.generic_form.get_url_field(form_descriptor))
if file_fields:
(field_index, field_descriptor) = file_fields.pop(0)
form_descriptor['field_index'] = field_index
return FormRequest(self.generic_form.get_value(field_descriptor),
meta=form_descriptor,
callback=self.parse_field_url_page,
dont_filter=True)
else:
return Request(url=form_descriptor.pop("form_url"),
meta=form_descriptor, callback=self.parse_form_page,
dont_filter=True)
def parse_field_url_page(self, response):
form_descriptor = response.request.meta
field_index = form_descriptor['field_index']
field_descriptor = form_descriptor['fields'][field_index]
self.generic_form.set_values_url_field(field_descriptor, response.body)
yield self.get_generic_form_start_request(form_descriptor)
def parse_form_page(self, response):
fill_form = self.generic_form.fill_generic_form
try:
for (args, url, method) in fill_form(response.url, response.body,
response.request.meta):
yield FormRequest(url, method=method, formdata=args,
callback=self.after_form_page,
dont_filter=True)
except Exception as e:
self.logger.warning(str(e))
for req in self._start_requests:
yield req
def after_form_page(self, response):
for result in self.parse(response):
yield result
def _get_allowed_domains(self, spec):
urls = [x['url'] for x in spec['templates']]
urls += [x['url'] for x in spec.get('init_requests', [])
if x['type'] == 'start']
urls += self._start_urls.allowed_domains
return [domain for scheme, domain in iter_unique_scheme_hostname(urls)]
def start_requests(self):
start_requests = []
if self.login_requests:
start_requests = self.login_requests
elif self.form_requests:
start_requests = self.form_requests