本文整理汇总了Python中classes.Registry.Registry.insert_mass方法的典型用法代码示例。如果您正苦于以下问题：Python Registry.insert_mass方法的具体用法？Python Registry.insert_mass怎么用？Python Registry.insert_mass使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类classes.Registry.Registry的用法示例。
在下文中一共展示了Registry.insert_mass方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: UrlsModel
# 需要导入模块: from classes.Registry import Registry [as 别名]
# 或者: from classes.Registry.Registry import insert_mass [as 别名]
class UrlsModel(object):
""" Class for work with urls table """
_db = None
def __init__(self):
self._db = Registry().get('ndb')
def add(
self, pid, host_id, url, referer='', response_code=0,
response_time=0, who_add='human', spidered=0, size=0, descr=''
):
""" Add url to table """
try:
return self._db.insert(
"urls",
{
"project_id": pid,
"host_id": host_id,
"hash": md5(url),
"url": url,
"referer": referer,
"response_code": response_code,
"response_time": response_time,
"when_add": int(time.time()),
"who_add": who_add,
"spidered": spidered,
"size": size,
"descr": descr
},
1
)
except BaseException as e:
if Registry().isset('logger'):
Registry().get('logger').ex(e)
else:
print "Can`t add url: " + str(e)
def add_mass(self, pid, host_id, data):
""" Add many urls at once in table """
to_insert = []
for row in data:
for field in ['url', 'referer', 'response_code', 'response_time', 'who_add', 'spidered', 'size', 'descr']:
if field not in row.keys():
if field in ['referer', 'response_code', 'response_time', 'descr']:
row[field] = ''
elif field in ['spidered', 'size']:
row[field] = 0
elif field == 'who_add':
row[field] = 'human'
elif field == 'url':
raise WSException("URL row must have a 'url' key")
for k in row.keys():
if k not in [
'url', 'referer', 'response_code', 'response_time', 'who_add', 'spidered', 'size', 'descr'
]:
raise WSException("Key '{0}' must not be in url data".format(k))
to_insert.append({
'project_id': pid,
"host_id": host_id,
"hash": md5(row['url']),
"url": row['url'],
"referer": row['referer'],
"response_code": row['response_code'],
"response_time": row['response_time'],
"when_add": int(time.time()),
"who_add": row['who_add'],
"spidered": row['spidered'],
"size": row['size'],
"descr": row['descr']
})
if len(to_insert)%50 == 0:
self._db.insert_mass("urls", to_insert, 1)
to_insert = []
if len(to_insert):
self._db.insert_mass("urls", to_insert, 1)
return True
def list_by_host_name(self, project_id, host, like=""):
""" Get urls list by host name and project_id """
host_id = HostsModel().get_id_by_name(project_id, host)
like_expr = "" \
if not len(like.strip()) \
else " AND url LIKE '%{0}%' ".format(self._db.escape(like.strip()))
return self._db.fetch_all(
"SELECT url, response_code as code, response_time as time, when_add, who_add, descr, size FROM urls "
"WHERE project_id = {0} AND host_id = {1} ".format(project_id, host_id) + like_expr +
"ORDER BY url"
)
def list_by_host_name_for_spider(self, project_id, host):
""" Get urls list by host name and project_id, but in special format for spider """
host_id = HostsModel().get_id_by_name(project_id, host)
#.........这里部分代码省略.........