Python leveldb.LevelDB方法代碼示例

# 需要導入模塊: import leveldb [as 別名]
# 或者: from leveldb import LevelDB [as 別名]
def text_to_db(from_text, to_db, split_char):
    '''Transfer text file to leveldb, return total and invalid count.'''
    total, invalid = 0, 0
    _split = split_char if split_char else ','
    _db = leveldb.LevelDB(to_db, create_if_missing=True) if isinstance(to_db, str) else to_db
    with open(from_text, 'r', encoding='utf-8') as _f:
        lines = _f.readlines()
        total = len(lines)
        for line in lines:
            if not line:
                invalid += 1
                continue
            # line = line.strip()
            if _split in line:
                _sub = line.split(_split, 1)
                _db.Put(_sub[0].encode('utf-8'), _sub[1].encode('utf-8'))
            else:
                _db.Put(line, '')
        return total, invalid 

# 需要導入模塊: import leveldb [as 別名]
# 或者: from leveldb import LevelDB [as 別名]
def excel_to_db(from_excel, to_db):
    '''Transfer Excel file to leveldb, return total count.'''
    _wb = load_workbook(from_excel, read_only=True)
    _ws = _wb.active
    _db = leveldb.LevelDB(to_db, create_if_missing=True) if isinstance(to_db, str) else to_db
    total = 0
    for _row in _ws.iter_rows(min_row=2, min_col=1, max_col=1):
        if _row and _row[0] and _row[1]:
            _key, _value = '', ''
            if _row[0].data_type == cell.Cell.TYPE_STRING:
                _key = _row[0].value.encode('utf-8')
                _key = ''.join(_key.split())
            if _row[1].data_type == cell.Cell.TYPE_STRING:
                _value = _row[0].value.encode('utf-8')
                _value = ''.join(_value.split())
            _db.Put(_key, _value)
            total += 1

    _wb.close()
    return total 

# 需要導入模塊: import leveldb [as 別名]
# 或者: from leveldb import LevelDB [as 別名]
def open(self):
        self._db = leveldb.LevelDB(self.db_path) 

# 需要導入模塊: import leveldb [as 別名]
# 或者: from leveldb import LevelDB [as 別名]
def open(self):
        self._db = leveldb.LevelDB(self.db_path, comparator=(
            'IntegerComparator', integer_comparator)) 

# 需要導入模塊: import leveldb [as 別名]
# 或者: from leveldb import LevelDB [as 別名]
def __init__(self, path, database_backend='leveldb'):
        if not (os.path.exists(path) or os.path.isdir(path)):
            err_msg = (
                '"{}" is not a valid directory').format(path)
            raise IOError(err_msg)

        if database_backend == 'unqlite':
            assert UNQLITE_AVAILABLE, (
                'You selected unqlite as database backend, but it is not '
                'installed. Please install it via `pip install unqlite`'
            )
            self.cui_db = unqlite.UnQLite(os.path.join(path, 'cui.unqlite'))
            self.cui_db_put = self.cui_db.store
            self.cui_db_get = self.cui_db.fetch
            self.semtypes_db = unqlite.UnQLite(os.path.join(path, 'semtypes.unqlite'))
            self.semtypes_db_put = self.semtypes_db.store
            self.semtypes_db_get = self.semtypes_db.fetch
        elif database_backend == 'leveldb':
            self.cui_db = leveldb.LevelDB(os.path.join(path, 'cui.leveldb'))
            self.cui_db_put = self.cui_db.Put
            self.cui_db_get = self.cui_db.Get
            self.semtypes_db = leveldb.LevelDB(os.path.join(path, 'semtypes.leveldb'))
            self.semtypes_db_put = self.semtypes_db.Put
            self.semtypes_db_get = self.semtypes_db.Get
        else:
            raise ValueError(f'database_backend {database_backend} not recognized') 

# 需要導入模塊: import leveldb [as 別名]
# 或者: from leveldb import LevelDB [as 別名]
def __init__(self, dbfile):
        self.uncommitted = dict()
        log.info('opening LevelDB', path=dbfile)
        self.db = leveldb.LevelDB(dbfile) 

# 需要導入模塊: import leveldb [as 別名]
# 或者: from leveldb import LevelDB [as 別名]
def __init__(self, db: leveldb.LevelDB, sync: bool):
        self._db = db
        self._batch = self._new_batch()
        self._sync = sync 

# 需要導入模塊: import leveldb [as 別名]
# 或者: from leveldb import LevelDB [as 別名]
def _new_db(self, path, **kwargs) -> leveldb.LevelDB:
        return leveldb.LevelDB(path, **kwargs) 

# 需要導入模塊: import leveldb [as 別名]
# 或者: from leveldb import LevelDB [as 別名]
def open(self):
        import leveldb
        self.db = leveldb.LevelDB(self.path) 

# 需要導入模塊: import leveldb [as 別名]
# 或者: from leveldb import LevelDB [as 別名]
def __init__(self, dbpath):
        self._map_db = leveldb.LevelDB(dbpath) 

# 需要導入模塊: import leveldb [as 別名]
# 或者: from leveldb import LevelDB [as 別名]
def dump(db_src):
    '''Dump database key and value items.'''
    _db = leveldb.LevelDB(db_src, create_if_missing=False) if isinstance(db_src, str) else db_src
    for _k, _v in _db.RangeIter():
        print(_k.decode(), _v.decode()) 

# 需要導入模塊: import leveldb [as 別名]
# 或者: from leveldb import LevelDB [as 別名]
def db_to_text(from_db, to_text):
    '''Transfer leveldb to text file.'''
    _db = leveldb.LevelDB(from_db, create_if_missing=False) if isinstance(from_db, str) else from_db
    with open(to_text, 'w', encoding='utf-8') as _f:
        for _k, _v in _db.RangeIter():
            _f.write(_k.decode() + ',' + _v.decode() + '\n') 

# 需要導入模塊: import leveldb [as 別名]
# 或者: from leveldb import LevelDB [as 別名]
def db_to_excel(from_db, to_excel):
    '''Transfer leveldb to Excel file, return total count.'''
    _db = leveldb.LevelDB(from_db, create_if_missing=False) if isinstance(from_db, str) else from_db
    _wb = Workbook()
    _ws = _wb.active
    total = 0
    for _k, _v in _db.RangeIter():
        _ws.append([_k.decode(), _v.decode()])
        total += 1
    _wb.save(to_excel)
    return total 

# 需要導入模塊: import leveldb [as 別名]
# 或者: from leveldb import LevelDB [as 別名]
def __init__(self, urlindex_file="", urls=None,
                 load_bad_url=False,
                 span_of_host=30,
                 max_in_mem=1024,
                 is_good_link=None):
        if not urlindex_file:
            urlindex_file = 'xcrawler.url.idx'
        if not is_good_link:
            import sys
            print 'no is_good_link function!!!!'
            sys.exit()
        self.is_good_link = is_good_link
        self.span_of_host = span_of_host
        self._urlindex = leveldb.LevelDB(urlindex_file)
        self._pool = {} # host: [urls]
        self._hosts_pop_recently = {}
        self.url_count = 0
        self.max_in_mem = max_in_mem
        self.last_load = time.time()
        if urls:
            self.url_count += len(urls)
            self.addmany(urls, always=True)
        self._load_from_url_index(load_bad_url, is_good_link)
        ## url is _URL_BAD if it has 5 times of 404
        self._404 = {}
        self._404_threshold = 5 

# 需要導入模塊: import leveldb [as 別名]
# 或者: from leveldb import LevelDB [as 別名]
def conn(self):
        """Open connection on LevelDB DataBase"""
        conn = DB("/tmp/{}.mining".format(self.conf.get("db")))
        return conn