Python lmdb.Environment方法代码示例

# 需要导入模块: import lmdb [as 别名]
# 或者: from lmdb import Environment [as 别名]
def begin_writer_txn(self, lmdbenv: lmdb.Environment,
                         buffer: bool = False) -> lmdb.Transaction:
        """Start a write enabled transaction on the given environment

        If multiple write transactions are requested for the same handle, only
        one instance of the transaction handle will be returened, and will not
        close until all operations on that handle have requested to close

        Parameters
        ----------
        lmdbenv : lmdb.Environment
            the environment to open the transaction on
        buffer : bool, optional
            if buffer objects should be used (the default is False, which does
            not use buffers)

        Returns
        -------
        lmdb.Transaction
            transaction handle to perform operations on
        """
        if self.WriterAncestors[lmdbenv] == 0:
            self.WriterTxn[lmdbenv] = lmdbenv.begin(write=True, buffers=buffer)
        self.WriterAncestors[lmdbenv] += 1
        return self.WriterTxn[lmdbenv] 

# 需要导入模块: import lmdb [as 别名]
# 或者: from lmdb import Environment [as 别名]
def begin_reader_txn(self, lmdbenv: lmdb.Environment,
                         buffer: bool = False) -> lmdb.Transaction:
        """Start a reader only txn for the given environment

        If there a read-only transaction for the same environment already exists
        then the same reader txn handle will be returned, and will not close
        until all operations on that handle have said they are finished.

        Parameters
        ----------
        lmdbenv : lmdb.Environment
            the environment to start the transaction in.
        buffer : bool, optional
            weather a buffer transaction should be used (the default is False,
            which means no buffers are returned)

        Returns
        -------
        lmdb.Transaction
            handle to the lmdb transaction.
        """
        if self.ReaderAncestors[lmdbenv] == 0:
            self.ReaderTxn[lmdbenv] = lmdbenv.begin(write=False, buffers=buffer)
        self.ReaderAncestors[lmdbenv] += 1
        return self.ReaderTxn[lmdbenv] 

# 需要导入模块: import lmdb [as 别名]
# 或者: from lmdb import Environment [as 别名]
def check_commit_hash_in_history(refenv, commit_hash):
    """Check if a commit hash exists in the repository history

    Parameters
    ----------
    refenv : lmdb.Environment
        refenv where the commit history is stored
    commit_hash : str
        hash of the commit to check for existence

    Returns
    -------
    bool
        True if exists, otherwise False
    """
    reftxn = TxnRegister().begin_reader_txn(refenv)
    try:
        commitParentKey = commit_parent_db_key_from_raw_key(commit_hash)
        commitParentVal = reftxn.get(commitParentKey, default=False)
        isCommitInHistory = True if commitParentVal is not False else False
    finally:
        TxnRegister().abort_reader_txn(refenv)
    return isCommitInHistory 

# 需要导入模块: import lmdb [as 别名]
# 或者: from lmdb import Environment [as 别名]
def tmp_cmt_env(refenv: lmdb.Environment, commit_hash: str):
    """create temporary unpacked lmdb environment from compressed structure

    Parameters
    ----------
    refenv : lmdb.Environment
        lmdb environment where the commit refs are stored
    commit_hash : str
        hash of the commit to get the contents of

    Returns
    -------
    lmdb.Environment
        environment with all db contents from ``commit`` unpacked
    """
    with tempfile.TemporaryDirectory() as tmpdir:
        tmpDF = os.path.join(tmpdir, 'test.lmdb')
        with closing(
                lmdb.open(tmpDF, sync=False, writemap=True, **LMDB_SETTINGS)
        ) as tmpDB:
            unpack_commit_ref(refenv, tmpDB, commit_hash)
            yield tmpDB 

# 需要导入模块: import lmdb [as 别名]
# 或者: from lmdb import Environment [as 别名]
def _commit_ref(stageenv: lmdb.Environment) -> DigestAndBytes:
    """Query and format all staged data records, and format it for ref storage.

    Parameters
    ----------
    stageenv : lmdb.Environment
        lmdb environment where the staged record data is actually stored.

    Returns
    -------
    DigestAndBytes
        Serialized and compressed version of all staged record data along with
        digest of commit refs.
    """
    from .queries import RecordQuery  # needed to avoid cyclic import

    querys = RecordQuery(dataenv=stageenv)
    allRecords = tuple(querys._traverse_all_records())
    res = commit_ref_db_val_from_raw_val(allRecords)
    return res


# -------------------- Format ref k/v pairs and write the commit to disk ---------------- 

# 需要导入模块: import lmdb [as 别名]
# 或者: from lmdb import Environment [as 别名]
def remove_stage_hash_records_from_hashenv(hashenv, stagehashenv):
    """Remove references to data additions during a hard reset

    For every hash record in stagehashenv, remove the corresponding k/v pair
    from the hashenv db. This is a dangerous operation if the stagehashenv was
    not appropriately constructed!!!

    Parameters
    ----------
    hashenv : lmdb.Environment
        db where all the permanent hash records are stored
    stagehashenv : lmdb.Environment
        db where all the staged hash records to be removed are stored.
    """
    stageHashKeys = HashQuery(stagehashenv).gen_all_hash_keys_db()
    hashtxn = TxnRegister().begin_writer_txn(hashenv)
    for hashKey in stageHashKeys:
        hashtxn.delete(hashKey)
    TxnRegister().commit_writer_txn(hashenv) 

# 需要导入模块: import lmdb [as 别名]
# 或者: from lmdb import Environment [as 别名]
def commit_hash_to_branch_name_map(branchenv: lmdb.Environment) -> dict:
    """Determine branch names which map to commit hashs

    Parameters
    ----------
    branchenv : lmdb.Environment
        db where the branch references are stored

    Returns
    -------
    dict
        keys are commit hash strings, values are list of branch names (strings)
        whose HEAD are at the key commit
    """
    outMap = defaultdict(list)
    branchNames = get_branch_names(branchenv=branchenv)
    for branchName in branchNames:
        branchHEAD = get_branch_head_commit(branchenv=branchenv, branch_name=branchName)
        outMap[branchHEAD].append(branchName)

    return outMap


# ----------------------------- Remotes --------------------------------------- 

# 需要导入模块: import lmdb [as 别名]
# 或者: from lmdb import Environment [as 别名]
def push_find_missing_schemas(self, commit, tmpDB: lmdb.Environment = None):

        if tmpDB is None:
            with tempfile.TemporaryDirectory() as tempD:
                tmpDF = os.path.join(tempD, 'test.lmdb')
                tmpDB = lmdb.open(path=tmpDF, **c.LMDB_SETTINGS)
                commiting.unpack_commit_ref(self.env.refenv, tmpDB, commit)
                c_schemaset = set(queries.RecordQuery(tmpDB).schema_hashes())
                c_schemas = list(c_schemaset)
                tmpDB.close()
        else:
            c_schemaset = set(queries.RecordQuery(tmpDB).schema_hashes())
            c_schemas = list(c_schemaset)

        request = hangar_service_pb2.FindMissingSchemasRequest()
        request.commit = commit
        request.schema_digests.extend(c_schemas)

        response = self.stub.PushFindMissingSchemas(request)
        return response 

# 需要导入模块: import lmdb [as 别名]
# 或者: from lmdb import Environment [as 别名]
def _diff3(a_env: lmdb.Environment,
               m_env: lmdb.Environment,
               d_env: lmdb.Environment) -> DiffAndConflictsDB:
        """Three way diff and conflict finder from ancestor, master, and dev commits.

        Parameters
        ----------
        a_env : lmdb.Environment
            unpacked lmdb environment for the ancestor commit
        m_env : lmdb.Environment
            unpacked lmdb environment for the master commit, current HEAD
        d_env : lmdb.Environment
            unpacked lmdb environment for the dev commit, compare to HEAD

        Returns
        -------
        DiffAndConflictsDB
            structure containing (`additions`, `deletions`, `mutations`) for
            diff, as well as the ConflictRecord struct.
        """
        it = ((a_env, m_env), (a_env, d_env), (d_env, m_env))
        diffs = tuple(starmap(diff_envs, it))  # significant perf improvement by map.
        conflict = find_conflicts(diffs[0], diffs[1])
        return DiffAndConflictsDB(diff=diffs[2], conflict=conflict) 

# 需要导入模块: import lmdb [as 别名]
# 或者: from lmdb import Environment [as 别名]
def __init__(self, filename, flag):
        """Constructor for the LMDBNoLockDatabase class.

        Args:
            filename (str): The filename of the database file.
            flag (str): a flag indicating the mode for opening the database.
                Refer to the documentation for anydbm.open().
        """
        super(LMDBNoLockDatabase, self).__init__()

        create = bool(flag == 'c')

        if flag == 'n':
            if os.path.isfile(filename):
                os.remove(filename)
            create = True

        self._lmdb = lmdb.Environment(
            path=filename,
            map_size=1024**4,
            map_async=True,
            writemap=True,
            readahead=False,
            subdir=False,
            create=create,
            lock=True)

    # pylint: disable=no-value-for-parameter 

# 需要导入模块: import lmdb [as 别名]
# 或者: from lmdb import Environment [as 别名]
def __init__(self):
        self.WriterAncestors = Counter()
        self.ReaderAncestors = Counter()
        self.WriterTxn: MutableMapping[lmdb.Environment, lmdb.Transaction] = {}
        self.ReaderTxn: MutableMapping[lmdb.Environment, lmdb.Transaction] = {} 

# 需要导入模块: import lmdb [as 别名]
# 或者: from lmdb import Environment [as 别名]
def commit_writer_txn(self, lmdbenv: lmdb.Environment) -> bool:
        """Commit changes made in a write-enable transaction handle

        As multiple objects can have references to the same open transaction handle,
        the data is not actually committed until all open transactions have called
        the commit method.

        Parameters
        ----------
        lmdbenv : lmdb.Environment
            the environment handle used to open the transaction

        Raises
        ------
        RuntimeError
            If the internal reference counting gets out of sync

        Returns
        -------
        bool
            True if this operation actually committed, otherwise false
            if other objects have references to the same (open) handle
        """
        ancestors = self.WriterAncestors[lmdbenv]
        if ancestors == 0:
            msg = f'hash ancestors are zero but commit called on {lmdbenv}'
            raise RuntimeError(msg)
        elif ancestors == 1:
            self.WriterTxn[lmdbenv].commit()
            self.WriterTxn.__delitem__(lmdbenv)
            ret = True
        else:
            ret = False
        self.WriterAncestors[lmdbenv] -= 1
        return ret 

# 需要导入模块: import lmdb [as 别名]
# 或者: from lmdb import Environment [as 别名]
def abort_reader_txn(self, lmdbenv: lmdb.Environment) -> bool:
        """Request to close a read-only transaction handle

        As multiple objects can have references to the same open transaction
        handle, the transaction is not actuall aborted until all open transactions
        have called the abort method


        Parameters
        ----------
        lmdbenv : lmdb.Environment
            the environment handle used to open the transaction

        Raises
        ------
        RuntimeError
            If the internal reference counting gets out of sync.

        Returns
        -------
        bool
            True if this operation actually aborted the transaction,
            otherwise False if other objects have references to the same (open)
            handle.
        """
        ancestors = self.ReaderAncestors[lmdbenv]
        if ancestors == 0:
            raise RuntimeError(f'hash ancestors are zero but abort called')
        elif ancestors == 1:
            self.ReaderTxn[lmdbenv].abort()
            self.ReaderTxn.__delitem__(lmdbenv)
            ret = True
        else:
            ret = False
        self.ReaderAncestors[lmdbenv] -= 1
        return ret 

# 需要导入模块: import lmdb [as 别名]
# 或者: from lmdb import Environment [as 别名]
def __init__(self, pth: Path):

        self.repo_path: Path = pth
        self.refenv: Optional[lmdb.Environment] = None
        self.hashenv: Optional[lmdb.Environment] = None
        self.stageenv: Optional[lmdb.Environment] = None
        self.branchenv: Optional[lmdb.Environment] = None
        self.stagehashenv: Optional[lmdb.Environment] = None
        self.cmtenv: MutableMapping[str, lmdb.Environment] = {}
        self._startup() 

# 需要导入模块: import lmdb [as 别名]
# 或者: from lmdb import Environment [as 别名]
def repo_is_initialized(self) -> bool:
        """Property to check if the repository is initialized, read-only attribute

        Returns
        -------
        bool
            True if repo environments are initialized, False otherwise
        """
        ret = True if isinstance(self.refenv, lmdb.Environment) else False
        return ret