Python sqlitedict.SqliteDict方法代码示例

# 需要导入模块: import sqlitedict [as 别名]
# 或者: from sqlitedict import SqliteDict [as 别名]
def __open_file(config_filename, force_write):
        """

        :param config_filename:
        :param force_write:
        :param quiet:
        :return:
        """
        if prompt_for_overwrite(config_filename, force_write):
            try:
                config_dirname = os.path.dirname(config_filename)
                if not os.path.isdir(config_dirname):
                    os.makedirs(config_dirname)
                if os.path.exists(config_filename):
                    os.remove(config_filename)
                return SqliteDict(config_filename)
            except Exception as e:
                print_exception(e)
        else:
            return None 

# 需要导入模块: import sqlitedict [as 别名]
# 或者: from sqlitedict import SqliteDict [as 别名]
def data(self, key=None):
        """
        Return the data at the requested key. Doesn't returns nested dictionaries and lists.
        If one of the value is a dictionary, it will return {'type': 'dict', 'keys': <Array of all the keys>}
        If one of the value is a list, it will return {'type': 'list', 'count': <number of elements in the list>}

        Can be found at GET /api/data?key=<KEY>
        :param key:                     Key of the requested information, separated by the character '¤'.
        :return:                        The data at the requested location stripped of its nested data.
        """
        result = self.get_item(self.results, key)
        # Returns only indexes or length if it's a complex type
        if isinstance(result, dict) or isinstance(result, SqliteDict):
            result = {'type': 'dict', 'keys': list(result.keys())}
        elif isinstance(result, list):
            result = {'type': 'list', 'length': len(result)}
        return {'data': result} 

# 需要导入模块: import sqlitedict [as 别名]
# 或者: from sqlitedict import SqliteDict [as 别名]
def persistence_update(p_dict, db_path=config.WN_FEATURE_CACHE_PATH):
    for dict_name in p_dict.keys():
        print("Updating Persistent WN Feature Dict:", dict_name)
        if dict_name != 'em_dict':
            in_db_dict = SqliteDict(str(db_path / dict_name),
                                    autocommit=False, tablename='the_table',
                                    flag='c')
            for key, v in tqdm(p_dict[dict_name].items()):
                if key not in in_db_dict:
                    in_db_dict[key] = v
            in_db_dict.commit()
            in_db_dict.close()

        elif dict_name == 'em_dict':
            in_db_dict = SqliteDict(str(db_path / dict_name),
                                    autocommit=False, tablename='the_table', flag='c')
            for key, v in tqdm(p_dict[dict_name].items()):
                if key not in in_db_dict:
                    in_db_dict[key] = v
            in_db_dict.commit()
            in_db_dict.close() 

# 需要导入模块: import sqlitedict [as 别名]
# 或者: from sqlitedict import SqliteDict [as 别名]
def iterative_build_raw_text():
    wiki_whole_db_cursor = get_cursor(str(config.WHOLE_WIKI_DB))
    wiki_whole_db_cursor.execute("SELECT * from unnamed")
    total_count = 0
    cur_count = 0

    with SqliteDict(str(config.WHOLE_WIKI_RAW_TEXT), encode=json.dumps, decode=json.loads) as whole_rindex_db:
        for key, value in tqdm(wiki_whole_db_cursor, total=TOTAL_ARTICLE_COUNT):
            cur_item = json.loads(value)
            raw_text = get_raw_text(cur_item)

            new_item = dict()
            new_item['title'] = cur_item['title']
            new_item['raw_text'] = raw_text
            whole_rindex_db[new_item['title']] = new_item

            cur_count += 1

            if cur_count % 5000 == 0:
                whole_rindex_db.commit()
                # break

        whole_rindex_db.commit()
        whole_rindex_db.close() 

# 需要导入模块: import sqlitedict [as 别名]
# 或者: from sqlitedict import SqliteDict [as 别名]
def __delitem__(self, key):
        if self.flag == 'r':
            raise RuntimeError('Refusing to delete from read-only SqliteDict')

        if key not in self:
            raise KeyError(key)
        assert isinstance(key, tuple) and len(key) == 2
        key1, key2 = key
        if key1 is not Ellipsis and key2 is Ellipsis:
            DEL_ITEM = 'DELETE FROM "%s" WHERE key1 = ?' % self.tablename
            del_args = (key1,)
        elif key1 is Ellipsis and key2 is not Ellipsis:
            DEL_ITEM = 'DELETE FROM "%s" WHERE key2 = ?' % self.tablename
            del_args = (key2,)
        elif key1 is not Ellipsis and key2 is not Ellipsis:
            DEL_ITEM = 'DELETE FROM "%s" WHERE key1 = ? AND key2 = ?' % self.tablename
            del_args = (key1, key2)
        else:
            raise ValueError('must provide at least one key')
        self.conn.execute(DEL_ITEM, del_args) 

# 需要导入模块: import sqlitedict [as 别名]
# 或者: from sqlitedict import SqliteDict [as 别名]
def load_from_file(self, config_type, config_path=None):
        if not config_path:
            config_path, _ = get_filename(config_type, self.report_name, self.report_dir)
        return SqliteDict(config_path, autocommit=True).data 

# 需要导入模块: import sqlitedict [as 别名]
# 或者: from sqlitedict import SqliteDict [as 别名]
def __init__(self, filename):
        """
        Constructor of the server object. Should not be called directly outside the class.

        :param filename:                Name of the file to write data to.
        :return:                        The server object.
        """
        self.results = SqliteDict(filename) 

# 需要导入模块: import sqlitedict [as 别名]
# 或者: from sqlitedict import SqliteDict [as 别名]
def page(self, key=None, page=None, pagesize=None):
        """
        Return a page of the data at the requested key. Doesn't returns nested dictionaries and lists.
        For example, if you set pagesize=10 and page=2, it should return element 10-19
        If one of the value is a dictionary, it will return {'type': 'dict', 'keys': <Array of all the keys>}
        If one of the value is a list, it will return {'type': 'list', 'count': <number of elements in the list>}

        Can be found at GET /api/page?key=<KEY>&page=<PAGE>&pagesize=<PAGESIZE>
        :param key:                     Key of the requested information, separated by the character '¤'.
        :param page:                    The number of the page you request.
        :param pagesize:                The size of the page you request.
        :return:                        A subset of the data at the requested location.
        """
        result = self.get_item(self.results, key)

        page = int(page)
        pagesize = int(pagesize)

        start = page * pagesize
        end = min((page + 1) * pagesize, len(result))

        if isinstance(result, dict) or isinstance(result, SqliteDict):
            page = {k: result.get(k) for k in sorted(list(result))[start:end]}
        if isinstance(result, list):
            page = result[start:end]

        return {'data': self.strip_nested_data(page)} 

# 需要导入模块: import sqlitedict [as 别名]
# 或者: from sqlitedict import SqliteDict [as 别名]
def _import_sql_data(data_dir):
    import sqlite3
    from sqlitedict import SqliteDict

    file_path = os.path.join(data_dir, DATA_FILE)

    # Find out what format we have
    with sqlite3.connect(file_path) as conn:
        try:
            conn.execute('select count(*) from zipgun_info')
            zipgun_info = SqliteDict(file_path, tablename='zipgun_info')
            version = zipgun_info.get('version', 0)
        except sqlite3.OperationalError:
            version = 0

    if version == 0:
        country_postal_codes = SqliteDict(file_path)
    elif version == 1:
        country_postal_codes = {}
        for country_code in zipgun_info['country_codes']:
            if country_code in country_postal_codes:
                raise ValueError('Duplicate entry found for {}'.format(
                    country_code))
            country_postal_codes[country_code] = SqliteDict(
                file_path, tablename='zg_{}'.format(country_code),
                journal_mode='OFF')
        zipgun_info.close()
    else:
        raise ValueError('Unknown data file version {}'.format(version))
    return country_postal_codes 

# 需要导入模块: import sqlitedict [as 别名]
# 或者: from sqlitedict import SqliteDict [as 别名]
def persistence_load(db_path=config.WN_FEATURE_CACHE_PATH):
    p_dict = {
        'hypernym_stems_dict': dict(),
        'hyponym_stems_dict': dict(),
        'hyper_lvl_dict': dict(),
        'hypo_lvl_dict': dict(),
        'ant_dict': dict(),
        'em_lemmas_dict': dict(),
    }

    # if em_dict:
    #     p_dict['em_dict'] = dict()

    for dict_name in p_dict.keys():
        print("Loading Persistent WN Feature Dict:", dict_name)
        if dict_name != 'em_dict':
            in_db_dict = SqliteDict(str(db_path / dict_name),
                                    autocommit=False, tablename='the_table',
                                    flag='c')
            for key, v in tqdm(in_db_dict.items()):
                p_dict[dict_name][key] = v
            in_db_dict.close()
        elif dict_name == 'em_dict':
            in_db_dict = SqliteDict(str(db_path / dict_name),
                                    autocommit=False, tablename='the_table', flag='c')
            for key, v in tqdm(in_db_dict.items()):
                p_dict[dict_name][key] = v
            in_db_dict.close()

    return p_dict 

# 需要导入模块: import sqlitedict [as 别名]
# 或者: from sqlitedict import SqliteDict [as 别名]
def get_cache(self, cache_name, autocommit=False):
        if cache_name not in self.caches:
            self.caches[cache_name] = SqliteDict(self.cache_file_path, tablename=cache_name, encode=json.dumps,
                                                 decode=json.loads, autocommit=autocommit)
        return self.caches[cache_name] 

# 需要导入模块: import sqlitedict [as 别名]
# 或者: from sqlitedict import SqliteDict [as 别名]
def __init__(self, cache_file_path):
        self.cache_file_path = cache_file_path
        self.caches = {
            'uploader_bans': SqliteDict(self.cache_file_path, tablename='uploader_bans', encode=json.dumps,
                                        decode=json.loads, autocommit=True),
            'syncer_bans': SqliteDict(self.cache_file_path, tablename='syncer_bans', encode=json.dumps,
                                      decode=json.loads, autocommit=True),
            'sa_bans': SqliteDict(self.cache_file_path,tablename='sa_bans',encode=json.dumps,decode=json.loads,autocommit=True)
        } 

# 需要导入模块: import sqlitedict [as 别名]
# 或者: from sqlitedict import SqliteDict [as 别名]
def iterative_abs_save_info(debug_num=None):
    total_doc_num = init_inspect.TOTAL_NUM_DOC if debug_num is None else debug_num
    cur_count = 0

    with open(config.ABS_WIKI_FILE, 'rb') as abs_file:
        with SqliteDict(str(config.ABS_PROCESS_FOR_RINDEX_DB), encode=json.dumps, decode=json.loads) as abs_rindex_db:
            for line in tqdm(abs_file, total=total_doc_num):
                item = json.loads(line)
                # print(item.keys())
                # print()
                if item['title'] in abs_rindex_db:
                    continue

                tokens, sent_offset = get_sentence_tokens(item['text'], item['charoffset'])
                poss = spacy_get_pos(tokens)
                assert len(tokens) == len(poss)
                # print(tokens)
                # print(sent_offset)
                abs_rindex_db[item['title']] = {
                    'tokens': tokens,
                    'poss': poss,
                    'sentence_offset': sent_offset
                }
                cur_count += 1

                if cur_count % 5000 == 0:
                    abs_rindex_db.commit()

            abs_rindex_db.commit()
            abs_rindex_db.close() 

# 需要导入模块: import sqlitedict [as 别名]
# 或者: from sqlitedict import SqliteDict [as 别名]
def write_to_db(file_path_list, out_file_path_name):
    with SqliteDict(str(out_file_path_name), encode=json.dumps, decode=json.loads) as db_dict:
        for i, file_path_name in tqdm(enumerate(file_path_list)):
            with bz2.open(file_path_name, mode='rb') as in_f:
                # print(f"Writing {i} file from", file_path_name, "to", out_file_path_name)
                for line in in_f:
                    item = json.loads(line)
                    title = item['title']
                    db_dict[title] = item

        db_dict.commit()
        db_dict.close() 

# 需要导入模块: import sqlitedict [as 别名]
# 或者: from sqlitedict import SqliteDict [as 别名]
def file_to_db(file_pathname, db_path_name):
    with open(file_pathname, encoding='utf-8', mode='r') as in_f, \
            SqliteDict(str(db_path_name), encode=json.dumps, decode=json.loads) as db_dict:
        for line in tqdm(in_f):
            item = json.loads(line)
            title = item['title']
            db_dict[title] = item

        db_dict.commit()
        db_dict.close()