Python OIBTree.keys方法代码示例

# 需要导入模块: from BTrees.OIBTree import OIBTree [as 别名]
# 或者: from BTrees.OIBTree.OIBTree import keys [as 别名]
class UUIDIndex(UnIndex):
    """Index for uuid fields with an unique value per key.

    The internal structure is:

    self._index = {datum:documentId]}
    self._unindex = {documentId:datum}

    For each datum only one documentId can exist.
    """

    meta_type = "UUIDIndex"

    manage_options = (
        {'label': 'Settings', 'action': 'manage_main'},
        {'label': 'Browse', 'action': 'manage_browse'},
    )

    query_options = ["query", "range"]

    manage = manage_main = DTMLFile('dtml/manageUUIDIndex', globals())
    manage_main._setName('manage_main')
    manage_browse = DTMLFile('../dtml/browseIndex', globals())

    def clear(self):
        self._length = Length()
        self._index = OIBTree()
        self._unindex = IOBTree()
        self._counter = Length()

    def numObjects(self):
        """Return the number of indexed objects. Since we have a 1:1 mapping
        from documents to values, we can reuse the stored length.
        """
        return self.indexSize()

    def uniqueValues(self, name=None, withLengths=0):
        """returns the unique values for name

        if withLengths is true, returns a sequence of
        tuples of (value, length)
        """
        if name is None:
            name = self.id
        elif name != self.id:
            raise StopIteration

        if not withLengths:
            for key in self._index.keys():
                yield key
        else:
            # We know the length for each value is one
            for key in self._index.keys():
                yield (key, 1)

    def insertForwardIndexEntry(self, entry, documentId):
        """Take the entry provided and put it in the correct place
        in the forward index.
        """
        if entry is None:
            return

        old_docid = self._index.get(entry, _marker)
        if old_docid is _marker:
            self._index[entry] = documentId
            self._length.change(1)
        elif old_docid != documentId:
            logger.error("A different document with value '%s' already "
                "exists in the index.'" % entry)

    def removeForwardIndexEntry(self, entry, documentId):
        """Take the entry provided and remove any reference to documentId
        in its entry in the index.
        """
        old_docid = self._index.get(entry, _marker)
        if old_docid is not _marker:
            del self._index[entry]
            self._length.change(-1)

    def _get_object_datum(self, obj, attr):
        # for a uuid it never makes sense to acquire a parent value via
        # Acquisition
        has_attr = getattr(aq_base(obj), attr, _marker)
        if has_attr is _marker:
            return _marker
        return super(UUIDIndex, self)._get_object_datum(obj, attr)

# 需要导入模块: from BTrees.OIBTree import OIBTree [as 别名]
# 或者: from BTrees.OIBTree.OIBTree import keys [as 别名]
class Lexicon(Persistent):
    """
    Implementation of :class:`zope.index.text.interfaces.ILexicon`.
    """

    def __init__(self, *pipeline):
        self._wids = OIBTree()  # word -> wid
        self._words = IOBTree() # wid -> word
        # wid 0 is reserved for words that aren't in the lexicon (OOV -- out
        # of vocabulary).  This can happen, e.g., if a query contains a word
        # we never saw before, and that isn't a known stopword (or otherwise
        # filtered out).  Returning a special wid value for OOV words is a
        # way to let clients know when an OOV word appears.
        self.wordCount = Length()
        self._pipeline = pipeline

    def wordCount(self):
        """Return the number of unique terms in the lexicon."""
        # overridden per instance
        return len(self._wids)

    def words(self):
        return self._wids.keys()

    def wids(self):
        return self._words.keys()

    def items(self):
        return self._wids.items()

    def sourceToWordIds(self, text):
        if text is None:
            text = ''
        last = _text2list(text)
        for element in self._pipeline:
            last = element.process(last)
        if not isinstance(self.wordCount, Length):
            # Make sure wordCount is overridden with a BTrees.Length.Length
            self.wordCount = Length(self.wordCount())
        # Strategically unload the length value so that we get the most
        # recent value written to the database to minimize conflicting wids
        # Because length is independent, this will load the most
        # recent value stored, regardless of whether MVCC is enabled
        self.wordCount._p_deactivate()
        return list(map(self._getWordIdCreate, last))

    def termToWordIds(self, text):
        last = _text2list(text)
        for element in self._pipeline:
            last = element.process(last)
        wids = []
        for word in last:
            wids.append(self._wids.get(word, 0))
        return wids

    def parseTerms(self, text):
        last = _text2list(text)
        for element in self._pipeline:
            process = getattr(element, "processGlob", element.process)
            last = process(last)
        return last

    def isGlob(self, word):
        return "*" in word or "?" in word

    def get_word(self, wid):
        return self._words[wid]

    def get_wid(self, word):
        return self._wids.get(word, 0)

    def globToWordIds(self, pattern):
        # Implement * and ? just as in the shell, except the pattern
        # must not start with either of these
        prefix = ""
        while pattern and pattern[0] not in "*?":
            prefix += pattern[0]
            pattern = pattern[1:]
        if not pattern:
            # There were no globbing characters in the pattern
            wid = self._wids.get(prefix, 0)
            if wid:
                return [wid]
            else:
                return []
        if not prefix:
            # The pattern starts with a globbing character.
            # This is too efficient, so we raise an exception.
            raise QueryError(
                "pattern %r shouldn't start with glob character" % pattern)
        pat = prefix
        for c in pattern:
            if c == "*":
                pat += ".*"
            elif c == "?":
                pat += "."
            else:
                pat += re.escape(c)
        pat += "$"
        prog = re.compile(pat)
#.........这里部分代码省略.........

# 需要导入模块: from BTrees.OIBTree import OIBTree [as 别名]
# 或者: from BTrees.OIBTree.OIBTree import keys [as 别名]
class Indexer(object):

    filestorage = database = connection = root = None

    def __init__(self, datafs, writable=0, trans=0, pack=0):
        self.trans_limit = trans
        self.pack_limit = pack
        self.trans_count = 0
        self.pack_count = 0
        self.stopdict = get_stopdict()
        self.mh = mhlib.MH()
        self.filestorage = FileStorage(datafs, read_only=(not writable))
        self.database = DB(self.filestorage)
        self.connection = self.database.open()
        self.root = self.connection.root()
        try:
            self.index = self.root["index"]
        except KeyError:
            self.index = self.root["index"] = TextIndexWrapper()
        try:
            self.docpaths = self.root["docpaths"]
        except KeyError:
            self.docpaths = self.root["docpaths"] = IOBTree()
        try:
            self.doctimes = self.root["doctimes"]
        except KeyError:
            self.doctimes = self.root["doctimes"] = IIBTree()
        try:
            self.watchfolders = self.root["watchfolders"]
        except KeyError:
            self.watchfolders = self.root["watchfolders"] = {}
        self.path2docid = OIBTree()
        for docid in self.docpaths.keys():
            path = self.docpaths[docid]
            self.path2docid[path] = docid
        try:
            self.maxdocid = max(self.docpaths.keys())
        except ValueError:
            self.maxdocid = 0
        print(len(self.docpaths), "Document ids")
        print(len(self.path2docid), "Pathnames")
        print(self.index.lexicon.length(), "Words")

    def dumpfreqs(self):
        lexicon = self.index.lexicon
        index = self.index.index
        assert isinstance(index, OkapiIndex)
        L = []
        for wid in lexicon.wids():
            freq = 0
            for f in index._wordinfo.get(wid, {}).values():
                freq += f
            L.append((freq, wid, lexicon.get_word(wid)))
        L.sort()
        L.reverse()
        for freq, wid, word in L:
            print("%10d %10d %s" % (wid, freq, word))

    def dumpwids(self):
        lexicon = self.index.lexicon
        index = self.index.index
        assert isinstance(index, OkapiIndex)
        for wid in lexicon.wids():
            freq = 0
            for f in index._wordinfo.get(wid, {}).values():
                freq += f
            print("%10d %10d %s" % (wid, freq, lexicon.get_word(wid)))

    def dumpwords(self):
        lexicon = self.index.lexicon
        index = self.index.index
        assert isinstance(index, OkapiIndex)
        for word in lexicon.words():
            wid = lexicon.get_wid(word)
            freq = 0
            for f in index._wordinfo.get(wid, {}).values():
                freq += f
            print("%10d %10d %s" % (wid, freq, word))

    def close(self):
        self.root = None
        if self.connection is not None:
            self.connection.close()
            self.connection = None
        if self.database is not None:
            self.database.close()
            self.database = None
        if self.filestorage is not None:
            self.filestorage.close()
            self.filestorage = None

    def interact(self, nbest=NBEST, maxlines=MAXLINES):
        try:
            import readline
        except ImportError:
            pass
        text = ""
        top = 0
        results = []
        while 1:
#.........这里部分代码省略.........

# 需要导入模块: from BTrees.OIBTree import OIBTree [as 别名]
# 或者: from BTrees.OIBTree.OIBTree import keys [as 别名]
class Lexicon(Persistent):

    implements(ILexicon)

    def __init__(self, *pipeline):
        self._wids = OIBTree()  # word -> wid
        self._words = IOBTree() # wid -> word
        # wid 0 is reserved for words that aren't in the lexicon (OOV -- out
        # of vocabulary).  This can happen, e.g., if a query contains a word
        # we never saw before, and that isn't a known stopword (or otherwise
        # filtered out).  Returning a special wid value for OOV words is a
        # way to let clients know when an OOV word appears.
        self._nextwid = 1
        self._pipeline = pipeline

        # Keep some statistics about indexing
        self._nbytes = 0 # Number of bytes indexed (at start of pipeline)
        self._nwords = 0 # Number of words indexed (after pipeline)

    def wordCount(self):
        """Return the number of unique terms in the lexicon."""
        return self._nextwid - 1

    def words(self):
        return self._wids.keys()

    def wids(self):
        return self._words.keys()

    def items(self):
        return self._wids.items()

    def sourceToWordIds(self, text):
        last = _text2list(text)
        for t in last:
            self._nbytes += len(t)
        for element in self._pipeline:
            last = element.process(last)
        self._nwords += len(last)
        return map(self._getWordIdCreate, last)

    def termToWordIds(self, text):
        last = _text2list(text)
        for element in self._pipeline:
            last = element.process(last)
        wids = []
        for word in last:
            wids.append(self._wids.get(word, 0))
        return wids

    def parseTerms(self, text):
        last = _text2list(text)
        for element in self._pipeline:
            process = getattr(element, "processGlob", element.process)
            last = process(last)
        return last

    def isGlob(self, word):
        return "*" in word or "?" in word

    def get_word(self, wid):
        return self._words[wid]

    def get_wid(self, word):
        return self._wids.get(word, 0)

    def globToWordIds(self, pattern):
        # Implement * and ? just as in the shell, except the pattern
        # must not start with either of these
        prefix = ""
        while pattern and pattern[0] not in "*?":
            prefix += pattern[0]
            pattern = pattern[1:]
        if not pattern:
            # There were no globbing characters in the pattern
            wid = self._wids.get(prefix, 0)
            if wid:
                return [wid]
            else:
                return []
        if not prefix:
            # The pattern starts with a globbing character.
            # This is too efficient, so we raise an exception.
            raise QueryError(
                "pattern %r shouldn't start with glob character" % pattern)
        pat = prefix
        for c in pattern:
            if c == "*":
                pat += ".*"
            elif c == "?":
                pat += "."
            else:
                pat += re.escape(c)
        pat += "$"
        prog = re.compile(pat)
        keys = self._wids.keys(prefix) # Keys starting at prefix
        wids = []
        for key in keys:
            if not key.startswith(prefix):
                break
#.........这里部分代码省略.........

# 需要导入模块: from BTrees.OIBTree import OIBTree [as 别名]
# 或者: from BTrees.OIBTree.OIBTree import keys [as 别名]
class Lexicon(Persistent):

    _v_nextid = None
    _wid_length_based = True  # Flag to distinguish new and old lexica

    def __init__(self, *pipeline):
        self.clear()
        self._pipeline = pipeline

    def clear(self):
        """Empty the lexicon.
        """
        self.length = Length()
        self._wid_length_based = False
        self._wids = OIBTree()  # word -> wid
        self._words = IOBTree()  # wid -> word
        # wid 0 is reserved for words that aren't in the lexicon (OOV -- out
        # of vocabulary).  This can happen, e.g., if a query contains a word
        # we never saw before, and that isn't a known stopword (or otherwise
        # filtered out).  Returning a special wid value for OOV words is a
        # way to let clients know when an OOV word appears.

    def length(self):
        """Return the number of unique terms in the lexicon.
        """
        # Overridden in instances with a BTrees.Length.Length
        raise NotImplementedError

    def words(self):
        return self._wids.keys()

    def wids(self):
        return self._words.keys()

    def items(self):
        return self._wids.items()

    def sourceToWordIds(self, text):
        last = _text2list(text)
        for element in self._pipeline:
            last = element.process(last)
        return list(map(self._getWordIdCreate, last))

    def termToWordIds(self, text):
        last = _text2list(text)
        for element in self._pipeline:
            process = getattr(element, "process_post_glob", element.process)
            last = process(last)
        wids = []
        for word in last:
            wids.append(self._wids.get(word, 0))
        return wids

    def parseTerms(self, text):
        last = _text2list(text)
        for element in self._pipeline:
            process = getattr(element, "processGlob", element.process)
            last = process(last)
        return last

    def isGlob(self, word):
        return "*" in word or "?" in word

    def get_word(self, wid):
        return self._words[wid]

    def get_wid(self, word):
        return self._wids.get(word, 0)

    def globToWordIds(self, pattern):
        # Implement * and ? just as in the shell, except the pattern
        # must not start with either of these
        prefix = ""
        while pattern and pattern[0] not in "*?":
            prefix += pattern[0]
            pattern = pattern[1:]
        if not pattern:
            # There were no globbing characters in the pattern
            wid = self._wids.get(prefix, 0)
            if wid:
                return [wid]
            else:
                return []
        if not prefix:
            # The pattern starts with a globbing character.
            # This is too efficient, so we raise an exception.
            raise QueryError(
                "pattern %r shouldn't start with glob character" % pattern)
        pat = prefix
        for c in pattern:
            if c == "*":
                pat += ".*"
            elif c == "?":
                pat += "."
            else:
                pat += re.escape(c)
        pat += "$"
        prog = re.compile(pat)
        keys = self._wids.keys(prefix)  # Keys starting at prefix
        wids = []
#.........这里部分代码省略.........

# 需要导入模块: from BTrees.OIBTree import OIBTree [as 别名]
# 或者: from BTrees.OIBTree.OIBTree import keys [as 别名]
class ContentTypeScopeManager(BTreeScopeManager):
    """
    A scope manager based on content types.

    This scope manager validates the request using the content type of
    the accessed object and the subpath of the request against a content
    type mapping.  The content type mapping to be used will be one of
    specified by the resource access key, the client key or default, and
    is resolved in this order.

    One more restriction imposed by this scope manager: mappings are
    enforced absolutely for access keys.  This allows clients to request
    new default scopes for themselves at will and/or have site-wide
    default scope changes without compromising the scopes already
    granted by the resource owner referenced by the access key.

    This however does not address the case where additional global
    restrictions that may be placed by the site owner as the focus is
    ultimately on the access keys.  Workaround is to revoke those keys
    and have the content owners issue new ones regardless of changes.

    Pruning of unused scope is not implemented.
    """

    zope.interface.implements(IContentTypeScopeManager)

    default_mapping_id = fieldproperty.FieldProperty(
        IContentTypeScopeManager['default_mapping_id'])

    def __init__(self):
        super(ContentTypeScopeManager, self).__init__()
        self._mappings = IOBTree()

        # Methods permitted to access this mapping with.  Originally
        # I wanted to provide alternative sets of mapping on a per
        # mapping_id basis, however this proved to be complex and
        # complicated due to extra relationships involved.
        self._methods = IOBTree()

        # For metadata related to the above.
        self._mappings_metadata = IOBTree()

        # To ease the usage of scopes, the mappings are referenced by
        # names and are called profiles which add a few useful fields to
        # allow slightly easier usage.  This separates the name from the
        # already active tokens such that once a token is instantiated
        # with a scope, the mapping is stuck until the token is revoked.
        self._named_mappings = OIBTree()  # name to id.

        # To not overburden the named mappings with work-in-progress
        # profiles, instantiate one here also.
        self._edit_mappings = OOBTree()

        self.default_mapping_id = self.addMapping({})

    # Main mapping related management methods.

    def addMapping(self, mapping, methods='GET HEAD OPTIONS', metadata=None):
        key = 0  # default?
        if len(self._mappings) > 0:
            # Can calculate the next key.
            key = self._mappings.maxKey() + 1
        self._mappings[key] = mapping
        self._methods[key] = methods.split()
        if metadata is not None:
            self._mappings_metadata[key] = metadata
        return key

    def getMapping(self, mapping_id, default=_marker):
        result = self._mappings.get(mapping_id, default)
        if result is _marker:
            raise KeyError()
        return result

    def getMappingMetadata(self, mapping_id, default=None):
        result = self._mappings_metadata.get(mapping_id, default)
        return result

    def getMappingId(self, name):
        # Returned ID could potentially not exist, what do?
        return self._named_mappings[name]

    def getMappingMethods(self, mapping_id, default=_marker):
        result = self._methods.get(mapping_id, default)
        if result is _marker:
            raise KeyError()
        return result

    def checkMethodPermission(self, mapping_id, method):
        methods = self.getMappingMethods(mapping_id, ())
        return method in methods

    def setMappingNameToId(self, name, mapping_id):
        self._named_mappings[name] = mapping_id

    def delMappingName(self, name):
        saved = self._named_mappings.pop(name, None)
        edits = self._edit_mappings.pop(name, None)
        return (saved, edits)

#.........这里部分代码省略.........

# 需要导入模块: from BTrees.OIBTree import OIBTree [as 别名]
# 或者: from BTrees.OIBTree.OIBTree import keys [as 别名]
class Lexicon(Persistent):
    """Maps words to word ids """

    __implements__ = LexiconInterface

    def __init__(self, truncate_left=0):
        self.truncate_left = truncate_left
        self.clear()

    def clear(self):
        self._nextid      = BTrees.Length.Length()
        self._forward_idx = OIBTree()
        self._inverse_idx = IOBTree()
        if self.truncate_left:
            self._lforward_idx = OIBTree()
        else:
            self._lforward_idx = None

    def getWordIdList(self, words):
        """ return a list of wordIds for a list of words """
    
        fw_idx = self._forward_idx
        fw_idx_get = fw_idx.get
        rev_idx = self._inverse_idx
        if self.truncate_left: lfw_idx = self._lforward_idx
        nextid = self._nextid

        wids = []
        append = wids.append

        for word in words:
            wid = fw_idx_get(word)
            if not wid:         
                nextid.change(1)
                wid = nextid()
                fw_idx[word] = wid
                rev_idx[wid] = word
                if self.truncate_left:
                    lfw_idx[word[::-1]] = wid
            append(wid)
        return wids

    def getWordId(self, word, default=None):
        """Return the matched word against the key."""
        return  self._forward_idx.get(word, default)

    def getWord(self, wid):
        """ return a word by its wid"""
        return self._inverse_idx[wid]

    def deleteWord(self, word):
        wid = self._forward_idx[word]
        del self._inverse_idx[wid]
        del self._forward_idx[word]

    def deleteWordId(self, wid):
        word = self._inverse_idx[wid]
        del self._forward_idx[word]
        del self._inverse_idx[wid]

    def getWordsForRightTruncation(self, prefix):
        """ Return a list for wordIds that match against prefix.
            We use the BTrees range search to perform the search
        """
        assert isinstance(prefix, unicode)
        return  self._forward_idx.keys(prefix, prefix + u'\uffff') 

    def getWordsForLeftTruncation(self, suffix):
        """ Return a sequence of word ids for a common suffix """
        suffix = suffix[::-1]
        assert isinstance(suffix, unicode)
        return  [w[::-1] for w in  self._lforward_idx.keys(suffix, suffix + u'\uffff') ] 

    def createRegex(self, pattern):
        """Translate a PATTERN to a regular expression """
        return '%s$' % pattern.replace( '*', '.*').replace( '?', '.')

    def getSimiliarWords(self, term, threshold=0.75): 
        """ return a list of similar words based on the levenshtein distance """
        return [ (w, ratio(w,term)) for w in self._forward_idx.keys() if ratio(w, term) > threshold  ]

    def getWordsForPattern(self, pattern):
        """ perform full pattern matching """

        # search for prefix in word
        mo = re.search('([\?\*])', pattern)
        if mo is None:
            return [ pattern ] 

        pos = mo.start(1)
        if pos==0: raise QueryParserError, \
            'word "%s" should not start with a globbing character' % pattern

        prefix = pattern[:pos]
        words = self._forward_idx.keys(prefix, prefix + u'\uffff')
        regex = re.compile( self.createRegex(pattern) )
        return [word  for word in words if regex.match(word) ] 

    def getWordsInRange(self, w1, w2):
        """ return all words within w1...w2 """
#.........这里部分代码省略.........