本文整理汇总了Python中validator.Validator.run方法的典型用法代码示例。如果您正苦于以下问题：Python Validator.run方法的具体用法？Python Validator.run怎么用？Python Validator.run使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类validator.Validator的用法示例。
在下文中一共展示了Validator.run方法的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: __init__
# 需要导入模块: from validator import Validator [as 别名]
# 或者: from validator.Validator import run [as 别名]
class ProxyPool:
def __init__(self):
self.sqlite = DatabaseObject(DB_CONFIG['SQLITE'])
self.Validator = Validator()
self.Crawler = Crawler()
def _monitor(self):
while True:
self._update(PROXYPOOL_CONFIG['UPDATE_TIME'])
self._delete(PROXYPOOL_CONFIG['DELETE_TIME'])
self._crawl(PROXYPOOL_CONFIG['CRAWL_TIME'])
time.sleep(1800)
def _crawl(self, minutes):
query = 'SELECT COUNT(*) FROM proxy WHERE updatetime>\'%s\'' % (
(datetime.datetime.now() - datetime.timedelta(minutes=minutes)).strftime('%Y-%m-%d %H:%M:%S'))
count = self.sqlite.executesql(query)[0]
if int(count[0]) < PROXYPOOL_CONFIG['MIN_IP_NUM']:
logger.info('Crawl proxy begin')
proxies = self.Crawler.run()
logger.info('Crawl proxy end')
logger.info('Validate proxy begin')
avaliable_proxies = self.Validator.run(proxies)
logger.info('Validate proxy end')
if DB_CONFIG['SQLITE']:
self.save2sqlite(avaliable_proxies)
def _delete(self, minutes):
query = 'DELETE FROM proxy WHERE updatetime<\'%s\'' % (
(datetime.datetime.now() - datetime.timedelta(minutes=minutes)).strftime('%Y-%m-%d %H:%M:%S'))
self.sqlite.executesql(query)
def _update(self, minutes):
query = 'SELECT ip,port FROM proxy WHERE updatetime<\'%s\'' % (
(datetime.datetime.now() - datetime.timedelta(minutes=minutes)).strftime('%Y-%m-%d %H:%M:%S'))
proxies = ['%s:%s' % n for n in self.sqlite.executesql(query)]
if proxies:
avaliable_proxies = self.Validator.run(proxies)
self.save2sqlite(avaliable_proxies)
def save2sqlite(self, result):
failed = self.sqlite.insert('proxy', result)
if failed:
failed = self.sqlite.update('proxy', failed)
if failed:
logger.info('Some ip failed to save: %s' % (str(failed)))
def _api(self):
ProxyServer(API_CONFIG['PORT'])
def run(self):
t1 = threading.Thread(target=self._api)
t2 = threading.Thread(target=self._monitor)
t1.start()
t2.start()示例2: load
# 需要导入模块: from validator import Validator [as 别名]
# 或者: from validator.Validator import run [as 别名]
def load(self, resource, resource_locator=None, skip_validation=False):
'''Load and validate ALP script'''
if resource is not None:
resource = resource_locator.find(resource)
# Detect circular includes
abspath = os.path.abspath(resource)
if abspath in self.__circular:
raise RuntimeError("Circular include detected (%s)" % abspath)
self.__circular.add(abspath)
# Load the file
root = script.parse_file(resource)
# Quick job: replace all includes with the proper
# included sub-files.
items = []
for entry in root.body:
if entry.ast_name == 'AlpInclude':
path = entry.path[1:-1]
included = self.load(path, resource_locator=resource_locator, skip_validation=True)
items.extend(included.body)
else:
items.append(entry)
root.body = root.body.replace(items)
if not skip_validation:
Validator.run(root)
return root示例3: Sniffer
# 需要导入模块: from validator import Validator [as 别名]
# 或者: from validator.Validator import run [as 别名]
class Sniffer(object):
def __init__(self, proxies):
self.proxies = sorted(proxies, key=lambda x: x.anonymity)
self.validator = Validator()
def verify(self):
result = {}
for level in config["anonymity"]:
proxies = self.classify().get(level, [])
logger.info("Validate [%s], total: %s", Proxy.anonymity.get(level), len(proxies))
result[level] = self.validator.run(proxies)
logger.info("Validate [%s], valid: %s", Proxy.anonymity.get(level), len(result[level]))
if config["save"]:
try:
self.save(config["save"], result)
except Exception as e:
logger.error("Exception: %s", e)
def save(self, name, result):
with open("./out/%s" % name, "wb") as f:
for level, proxies in result.iteritems():
f.writelines(
[
"[%s]%s\t%s\n" % (level, line[0], line[1])
for line in sorted(proxies.items(), key=operator.itemgetter(1))
]
)
def classify(self):
group = {}
for k, v in groupby(self.proxies, key=lambda x: x.anonymity):
group[k] = list(set(v))
return group示例4: __init__
# 需要导入模块: from validator import Validator [as 别名]
# 或者: from validator.Validator import run [as 别名]
class ProxyPool:
def __init__(self):
self.sqlite = DatabaseObject(DB_CONFIG['SQLITE'])
self.Validator = Validator()
self.Crawler = Crawler()
def _monitor(self):
while True:
self._update(PROXYPOOL_CONFIG['UPDATE_TIME'])
self._delete(PROXYPOOL_CONFIG['DELETE_TIME'], PROXYPOOL_CONFIG['DELETE_SCORE'])
self._crawl(PROXYPOOL_CONFIG['CRAWL_TIME'])
time.sleep(PROXYPOOL_CONFIG['SLEEP_TIME'])
def _crawl(self, minutes):
query = "SELECT COUNT(*) FROM proxy WHERE updatetime>'%s'" % \
((datetime.datetime.now() - datetime.timedelta(minutes=minutes)).strftime('%Y-%m-%d %H:%M:%S'))
count = self.sqlite.executesql(query)[0]
if int(count[0]) < PROXYPOOL_CONFIG['MIN_IP_NUM']:
logger.info('Crawl proxy begin')
proxies = self.Crawler.run()
logger.info('Crawl proxy end')
logger.info('Validate proxy begin')
proxies = [(n, '') for n in proxies]
avaliable_proxies = self.Validator.run(proxies)
logger.info('Validate proxy end')
if DB_CONFIG['SQLITE']:
self.save2sqlite(avaliable_proxies)
time.sleep(600)
self._crawl(minutes)
else:
return
def _delete(self, minutes, score):
query = "DELETE FROM proxy WHERE updatetime<'%s' or score<%s" % (
(datetime.datetime.now() - datetime.timedelta(minutes=minutes)).strftime('%Y-%m-%d %H:%M:%S'), score)
self.sqlite.executesql(query)
def _update(self, minutes):
query = "SELECT ip,port,protocol FROM proxy WHERE updatetime<'%s'" % (
(datetime.datetime.now() - datetime.timedelta(minutes=minutes)).strftime('%Y-%m-%d %H:%M:%S'))
proxies = [('%s:%s' % n[:2], n[2]) for n in self.sqlite.executesql(query)]
if proxies:
avaliable_proxies = self.Validator.run(proxies)
validated_fail_proxies = list(set([tuple(n[0].split(':')) for n in proxies]).difference(set([(n['ip'], n['port']) for n in avaliable_proxies])))
self.save2sqlite(avaliable_proxies)
self._minus_score(validated_fail_proxies)
def save2sqlite(self, result):
failed = self.sqlite.insert('proxy', result)
if failed:
failed = self.sqlite.update('proxy', failed)
if failed:
logger.info('Some ip failed to save: %s' % (str(failed)))
def _minus_score(self, result):
query = "UPDATE proxy SET score=(score-1) WHERE ip=? AND port=?;"
self.sqlite.cursor.executemany(query, result)
self.sqlite.db.commit()
def _api(self):
ProxyServer(API_CONFIG['PORT'])
def run(self):
t1 = threading.Thread(target=self._api)
t2 = threading.Thread(target=self._monitor)
t1.start()
t2.start()