本文整理汇总了Python中pattern.Pattern.convertPattern方法的典型用法代码示例。如果您正苦于以下问题：Python Pattern.convertPattern方法的具体用法？Python Pattern.convertPattern怎么用？Python Pattern.convertPattern使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类pattern.Pattern的用法示例。
在下文中一共展示了Pattern.convertPattern方法的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: __fetch_requests
# 需要导入模块: from pattern import Pattern [as 别名]
# 或者: from pattern.Pattern import convertPattern [as 别名]
def __fetch_requests(self, task_entry):
p = Pattern(task_entry, self.__getCurrentShell(task_entry), self.global_data)
timeout = task_entry.get('timeout', 120)
urls = p.convertPattern('url')
s = requests.session()
headers = task_entry.get('headers', [])
task_entry['datas'] = []
if not urls:
return task_entry
for url in urls:
self.logger.info("fetching " + url)
data = ""
if not url:
# do not fetch null url
continue
try:
response = s.get(url, timeout=timeout, headers=headers)
if 200 != response.status_code:
self.logger.error("fetch " + url + " failed with code " + (str)(response.status_code))
data = response.text
except:
self.logger.error("fetch " + url + " failed in sockets")
task_entry['datas'].append(data)
return task_entry示例2: __fetch
# 需要导入模块: from pattern import Pattern [as 别名]
# 或者: from pattern.Pattern import convertPattern [as 别名]
def __fetch(self, task_entry):
p = Pattern(task_entry, self.__getCurrentShell(task_entry), self.config_data)
urls = p.convertPattern('url')
s = requests.session()
task_entry['datas'] = []
for url in urls:
self.logger.info("fetching " + url)
if "" == url:
# do not fetch null url
continue
response = s.get(url)
if 200 != response.status_code:
self.logger.error("fetch " + url + " failed with code" + response.status_code)
data = response.text
task_entry['datas'].append(data)
return task_entry示例3: __fetch_webkit
# 需要导入模块: from pattern import Pattern [as 别名]
# 或者: from pattern.Pattern import convertPattern [as 别名]
def __fetch_webkit(self, task_entry):
p = Pattern(task_entry, self.__getCurrentShell(task_entry), self.global_data)
import cwebbrowser
task_entry['datas'] = []
urls = p.convertPattern('url')
timeout = task_entry.get('timeout', 120)
delay = task_entry.get('delay', 0)
for url in urls:
self.logger.info("fetching " + url)
data = ""
if not url:
# do not fetch null url
continue
browser = cwebbrowser.CWebBrowser()
browser.setHeaders(task_entry.get('headers', []))
#browser.show();
try:
browser.load(url=url, load_timeout=timeout, delay=delay)
except cwebbrowser.Timeout:
self.logger.error("fetch " + url + " timeout ")
except Exception, exception:
self.logger.error("fetch " + url + " error ")
print "Exception message:", exception
else:
html = browser.html()
if html:
html = html.encode('utf-8')
data = html
else:
self.logger.error("fetch " + url + " failed with no response")
task_entry['datas'].append(data)
browser.close()