Python writing.AsyncWriter方法代码示例

# 需要导入模块: from whoosh import writing [as 别名]
# 或者: from whoosh.writing import AsyncWriter [as 别名]
def test_asyncwriter():
    schema = fields.Schema(id=fields.ID(stored=True), text=fields.TEXT)
    with TempIndex(schema, "asyncwriter") as ix:
        domain = (u("alfa"), u("bravo"), u("charlie"), u("delta"), u("echo"),
                  u("foxtrot"), u("golf"), u("hotel"), u("india"))

        writers = []
        # Simulate doing 20 (near-)simultaneous commits. If we weren't using
        # AsyncWriter, at least some of these would fail because the first
        # writer wouldn't be finished yet.
        for i in xrange(20):
            w = writing.AsyncWriter(ix)
            writers.append(w)
            w.add_document(id=text_type(i),
                           text=u(" ").join(random.sample(domain, 5)))
            w.commit()

        # Wait for all writers to finish before checking the results
        for w in writers:
            if w.running:
                w.join()

        # Check whether all documents made it into the index.
        with ix.reader() as r:
            assert sorted([int(id) for id in r.lexicon("id")]) == list(range(20)) 

# 需要导入模块: from whoosh import writing [as 别名]
# 或者: from whoosh.writing import AsyncWriter [as 别名]
def test_asyncwriter_no_stored():
    schema = fields.Schema(id=fields.ID, text=fields.TEXT)
    with TempIndex(schema, "asyncnostored") as ix:
        domain = (u("alfa"), u("bravo"), u("charlie"), u("delta"), u("echo"),
                  u("foxtrot"), u("golf"), u("hotel"), u("india"))

        writers = []
        # Simulate doing 20 (near-)simultaneous commits. If we weren't using
        # AsyncWriter, at least some of these would fail because the first
        # writer wouldn't be finished yet.
        for i in xrange(20):
            w = writing.AsyncWriter(ix)
            writers.append(w)
            w.add_document(id=text_type(i),
                           text=u(" ").join(random.sample(domain, 5)))
            w.commit()

        # Wait for all writers to finish before checking the results
        for w in writers:
            if w.running:
                w.join()

        # Check whether all documents made it into the index.
        with ix.reader() as r:
            assert sorted([int(id) for id in r.lexicon("id")]) == list(range(20)) 

# 需要导入模块: from whoosh import writing [as 别名]
# 或者: from whoosh.writing import AsyncWriter [as 别名]
def add_to_index(self, item_id, text):
        from whoosh.writing import AsyncWriter
        writer = AsyncWriter(self.ix)
        writer.update_document(id=item_id, text=text.lower())
        writer.commit() 

# 需要导入模块: from whoosh import writing [as 别名]
# 或者: from whoosh.writing import AsyncWriter [as 别名]
def remove(self, item_id):
        from whoosh.writing import AsyncWriter
        writer = AsyncWriter(self.ix)
        writer.delete_by_term('id', item_id)
        writer.commit() 

# 需要导入模块: from whoosh import writing [as 别名]
# 或者: from whoosh.writing import AsyncWriter [as 别名]
def update(self, index, iterable, commit=True):
        if not self.setup_complete:
            self.setup()

        self.index = self.index.refresh()
        writer = AsyncWriter(self.index)

        for obj in iterable:
            try:
                doc = index.full_prepare(obj)
            except SkipDocument:
                self.log.debug(u"Indexing for object `%s` skipped", obj)
            else:
                # Really make sure it's unicode, because Whoosh won't have it any
                # other way.
                for key in doc:
                    doc[key] = self._from_python(doc[key])

                # Document boosts aren't supported in Whoosh 2.5.0+.
                if 'boost' in doc:
                    del doc['boost']

                try:
                    writer.update_document(**doc)
                except Exception as e:
                    if not self.silently_fail:
                        raise

                    # We'll log the object identifier but won't include the actual object
                    # to avoid the possibility of that generating encoding errors while
                    # processing the log message:
                    self.log.error(u"%s while preparing object for update" % e.__class__.__name__,
                                   exc_info=True, extra={"data": {"index": index,
                                                                  "object": get_identifier(obj)}})

        if len(iterable) > 0:
            # For now, commit no matter what, as we run into locking issues otherwise.
            writer.commit() 

# 需要导入模块: from whoosh import writing [as 别名]
# 或者: from whoosh.writing import AsyncWriter [as 别名]
def get_writer(self):
        """
        Return an object which can be used to write to the search index
        The object should deal with file locking from across threads and
        processes although writes will happen asyncronously
        """
        index = self.get_index()
        return AsyncWriter(index) 

# 需要导入模块: from whoosh import writing [as 别名]
# 或者: from whoosh.writing import AsyncWriter [as 别名]
def turnOnWriteMode(self):
        self.isWriteModeOn = True
        self.isReadModeOn = False
        self.schema = self.getSchema()
        if not os.path.exists(self.directory):
            logging.info("directory does not exist")
            os.mkdir(self.directory)
            self.ix = create_in(self.directory, self.schema)
        self.ix = open_dir(self.directory)
        # self.writer = AsyncWriter(self.ix)
        self.writer = self.ix.writer()

    # def getIsNeedParse(self):
    #     return 

# 需要导入模块: from whoosh import writing [as 别名]
# 或者: from whoosh.writing import AsyncWriter [as 别名]
def _post_flush(app, changes):
    by_type = defaultdict(list)

    for instance, change in changes:
        update = change in UPDATE_FIELDS

        if hasattr(instance.__class__, '__searchable__'):
            by_type[instance.__class__].append((update, instance))

    procs = app.config.get('WHOOSH_INDEXING_CPUS', 2)
    limit = app.config.get('WHOOSH_INDEXING_RAM', 256)

    for model_name, values in by_type.items():
        ref = values[0][1]

        index = AsyncWriter(search_index(app, ref),
                            delay=0.15,
                            writerargs=dict(proc=procs, limitmb=limit))
        primary_field = ref.whoosh.pk
        searchable = ref.__searchable__

        with index as writer:
            for do_update, v in values:
                pk = str(getattr(v, primary_field))

                if do_update:
                    # get all the columns defined in `__searchable__`
                    attrs = {}
                    for k in searchable:
                        try:
                            attrs[k] = str(getattr(v, k))
                        except AttributeError:
                            raise AttributeError('invalid attribute `%s`' % k)

                    attrs[primary_field] = pk

                    # create a new document, or update an old one, with all
                    #  of our new column values
                    writer.update_document(**attrs)
                else:
                    # remove the document by field `primary field` value `pk`
                    writer.delete_by_term(primary_field, pk)
    return EXT_CONTINUE 

# 需要导入模块: from whoosh import writing [as 别名]
# 或者: from whoosh.writing import AsyncWriter [as 别名]
def __init__(self, mode='read', directory=INDEX_DIRECTORY):
        """Creates an Indexing object to communicate with Woosh.

        Args:
            mode: str (default: "read"), "read" or "write" (the mode to use).
            directory: str, where to index files (defaults to INDEX_DIRECTORY).
        """

        # self.schema = self.getSchema()
        # if not os.path.exists("indexdir"):
        #     print ("directory not exist")
        #     os.mkdir("indexdir")
        #     self.ix = create_in("indexdir", self.schema)
        # self.ix = open_dir("indexdir")
        # self.writer = AsyncWriter(self.ix)
        # self.writer = self.ix.writer()
        # self.ix.reader()
        self.directory = directory
        self.isWriteModeOn = False
        self.isReadModeOn = False

        # Loads stopwords from the associated file.
        with open(STOPWORDS_FILE, 'r') as f:
            self.stoplist = set(f.read().strip().split())

        with open(BAD_WORDS_FILE, 'r') as f:
            self.bad_words = set(f.read().strip().split())

        mode = mode.lower()
        if mode == 'write':
            self.turnOnWriteMode()
        elif mode == 'read':
            self.turnOnReadMode()

            # Initializes the parsers.
            self.question_parser = QueryParser('ba',
                                               schema=self.ix.schema,
                                               group=syntax.OrGroup)
            self.answer_parser = QueryParser('title',
                                             schema=self.ix.schema,
                                             group=syntax.OrGroup)
            self.searcher = self.ix.searcher()
        else:
            raise ValueError('Invalid mode: "%s" (should be "read" or '
                             '"write").' % mode)