本文整理汇总了Python中leveldb.LevelDB方法的典型用法代码示例。如果您正苦于以下问题:Python leveldb.LevelDB方法的具体用法?Python leveldb.LevelDB怎么用?Python leveldb.LevelDB使用的例子?那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类leveldb
的用法示例。
在下文中一共展示了leveldb.LevelDB方法的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: text_to_db
# 需要导入模块: import leveldb [as 别名]
# 或者: from leveldb import LevelDB [as 别名]
def text_to_db(from_text, to_db, split_char):
'''Transfer text file to leveldb, return total and invalid count.'''
total, invalid = 0, 0
_split = split_char if split_char else ','
_db = leveldb.LevelDB(to_db, create_if_missing=True) if isinstance(to_db, str) else to_db
with open(from_text, 'r', encoding='utf-8') as _f:
lines = _f.readlines()
total = len(lines)
for line in lines:
if not line:
invalid += 1
continue
# line = line.strip()
if _split in line:
_sub = line.split(_split, 1)
_db.Put(_sub[0].encode('utf-8'), _sub[1].encode('utf-8'))
else:
_db.Put(line, '')
return total, invalid
示例2: excel_to_db
# 需要导入模块: import leveldb [as 别名]
# 或者: from leveldb import LevelDB [as 别名]
def excel_to_db(from_excel, to_db):
'''Transfer Excel file to leveldb, return total count.'''
_wb = load_workbook(from_excel, read_only=True)
_ws = _wb.active
_db = leveldb.LevelDB(to_db, create_if_missing=True) if isinstance(to_db, str) else to_db
total = 0
for _row in _ws.iter_rows(min_row=2, min_col=1, max_col=1):
if _row and _row[0] and _row[1]:
_key, _value = '', ''
if _row[0].data_type == cell.Cell.TYPE_STRING:
_key = _row[0].value.encode('utf-8')
_key = ''.join(_key.split())
if _row[1].data_type == cell.Cell.TYPE_STRING:
_value = _row[0].value.encode('utf-8')
_value = ''.join(_value.split())
_db.Put(_key, _value)
total += 1
_wb.close()
return total
示例3: open
# 需要导入模块: import leveldb [as 别名]
# 或者: from leveldb import LevelDB [as 别名]
def open(self):
self._db = leveldb.LevelDB(self.db_path)
示例4: open
# 需要导入模块: import leveldb [as 别名]
# 或者: from leveldb import LevelDB [as 别名]
def open(self):
self._db = leveldb.LevelDB(self.db_path, comparator=(
'IntegerComparator', integer_comparator))
示例5: __init__
# 需要导入模块: import leveldb [as 别名]
# 或者: from leveldb import LevelDB [as 别名]
def __init__(self, path, database_backend='leveldb'):
if not (os.path.exists(path) or os.path.isdir(path)):
err_msg = (
'"{}" is not a valid directory').format(path)
raise IOError(err_msg)
if database_backend == 'unqlite':
assert UNQLITE_AVAILABLE, (
'You selected unqlite as database backend, but it is not '
'installed. Please install it via `pip install unqlite`'
)
self.cui_db = unqlite.UnQLite(os.path.join(path, 'cui.unqlite'))
self.cui_db_put = self.cui_db.store
self.cui_db_get = self.cui_db.fetch
self.semtypes_db = unqlite.UnQLite(os.path.join(path, 'semtypes.unqlite'))
self.semtypes_db_put = self.semtypes_db.store
self.semtypes_db_get = self.semtypes_db.fetch
elif database_backend == 'leveldb':
self.cui_db = leveldb.LevelDB(os.path.join(path, 'cui.leveldb'))
self.cui_db_put = self.cui_db.Put
self.cui_db_get = self.cui_db.Get
self.semtypes_db = leveldb.LevelDB(os.path.join(path, 'semtypes.leveldb'))
self.semtypes_db_put = self.semtypes_db.Put
self.semtypes_db_get = self.semtypes_db.Get
else:
raise ValueError(f'database_backend {database_backend} not recognized')
示例6: __init__
# 需要导入模块: import leveldb [as 别名]
# 或者: from leveldb import LevelDB [as 别名]
def __init__(self, dbfile):
self.uncommitted = dict()
log.info('opening LevelDB', path=dbfile)
self.db = leveldb.LevelDB(dbfile)
示例7: __init__
# 需要导入模块: import leveldb [as 别名]
# 或者: from leveldb import LevelDB [as 别名]
def __init__(self, db: leveldb.LevelDB, sync: bool):
self._db = db
self._batch = self._new_batch()
self._sync = sync
示例8: _new_db
# 需要导入模块: import leveldb [as 别名]
# 或者: from leveldb import LevelDB [as 别名]
def _new_db(self, path, **kwargs) -> leveldb.LevelDB:
return leveldb.LevelDB(path, **kwargs)
示例9: open
# 需要导入模块: import leveldb [as 别名]
# 或者: from leveldb import LevelDB [as 别名]
def open(self):
import leveldb
self.db = leveldb.LevelDB(self.path)
示例10: __init__
# 需要导入模块: import leveldb [as 别名]
# 或者: from leveldb import LevelDB [as 别名]
def __init__(self, dbpath):
self._map_db = leveldb.LevelDB(dbpath)
示例11: dump
# 需要导入模块: import leveldb [as 别名]
# 或者: from leveldb import LevelDB [as 别名]
def dump(db_src):
'''Dump database key and value items.'''
_db = leveldb.LevelDB(db_src, create_if_missing=False) if isinstance(db_src, str) else db_src
for _k, _v in _db.RangeIter():
print(_k.decode(), _v.decode())
示例12: db_to_text
# 需要导入模块: import leveldb [as 别名]
# 或者: from leveldb import LevelDB [as 别名]
def db_to_text(from_db, to_text):
'''Transfer leveldb to text file.'''
_db = leveldb.LevelDB(from_db, create_if_missing=False) if isinstance(from_db, str) else from_db
with open(to_text, 'w', encoding='utf-8') as _f:
for _k, _v in _db.RangeIter():
_f.write(_k.decode() + ',' + _v.decode() + '\n')
示例13: db_to_excel
# 需要导入模块: import leveldb [as 别名]
# 或者: from leveldb import LevelDB [as 别名]
def db_to_excel(from_db, to_excel):
'''Transfer leveldb to Excel file, return total count.'''
_db = leveldb.LevelDB(from_db, create_if_missing=False) if isinstance(from_db, str) else from_db
_wb = Workbook()
_ws = _wb.active
total = 0
for _k, _v in _db.RangeIter():
_ws.append([_k.decode(), _v.decode()])
total += 1
_wb.save(to_excel)
return total
示例14: __init__
# 需要导入模块: import leveldb [as 别名]
# 或者: from leveldb import LevelDB [as 别名]
def __init__(self, urlindex_file="", urls=None,
load_bad_url=False,
span_of_host=30,
max_in_mem=1024,
is_good_link=None):
if not urlindex_file:
urlindex_file = 'xcrawler.url.idx'
if not is_good_link:
import sys
print 'no is_good_link function!!!!'
sys.exit()
self.is_good_link = is_good_link
self.span_of_host = span_of_host
self._urlindex = leveldb.LevelDB(urlindex_file)
self._pool = {} # host: [urls]
self._hosts_pop_recently = {}
self.url_count = 0
self.max_in_mem = max_in_mem
self.last_load = time.time()
if urls:
self.url_count += len(urls)
self.addmany(urls, always=True)
self._load_from_url_index(load_bad_url, is_good_link)
## url is _URL_BAD if it has 5 times of 404
self._404 = {}
self._404_threshold = 5
示例15: conn
# 需要导入模块: import leveldb [as 别名]
# 或者: from leveldb import LevelDB [as 别名]
def conn(self):
"""Open connection on LevelDB DataBase"""
conn = DB("/tmp/{}.mining".format(self.conf.get("db")))
return conn