Python index.writer函数代码示例

    def update(self, force_rebuild=False):
        """ Adds/updates all items in repo to index. Note: querying will call this automatically."""

        # if we've already updated the index during this script run, we're done!
        if self.index_updated:
            return False

        # if the index is not based on the current commit, rebuild from scratch
        if not self.index_based_on_current_commit():
            force_rebuild = True

        if force_rebuild:
            # get a new clean/empty index
            index = self.get_index(force_rebuild)
            index_writer = index.writer()

            # index all documents
            documents = self.document_iterator()
            activity_description = 'Rebuilding'
        else:
            # use the current index
            index = self.get_index()
            index_writer = index.writer()

            # delete uncommitted files that are in index already
            for filepath in self.get_indexed_uncommitted_files():
                index_writer.delete_by_term('path', filepath)

            # get list of uncommitted files and persist it
            uncommitted_files = lib_git.get_uncommitted_oval()
            self.set_indexed_uncommitted_files(uncommitted_files)

            # if there are no uncommitted files to index, we're done
            if not uncommitted_files:
                index_writer.commit()
                return False

            # index only uncommitted files
            documents = self.document_iterator(uncommitted_files)
            activity_description = 'Updating'

        # add all definition files to index
        counter = 0
        for document in documents:
            counter = counter + 1
            self.status_spinner(counter, '{0} {1} index'.format(activity_description, self.index_name), self.item_label)
            if 'deleted' in document and document['deleted']:
                index_writer.delete_by_term('path', document['path'])
                #self.message('debug', 'Deleting from index:\n\t{0} '.format(document['path']))
            else:
                index_writer.add_document(**document)
                #self.message('debug', 'Upserting to index:\n\t{0} '.format(document['path']))
        index_writer.commit()
        self.status_spinner(counter, '{0} {1} index'.format(activity_description, self.index_name), self.item_label, True)

        # update indexed commit
        self.set_indexed_commit_hash()
        self.index_updated = True

 def _get_index(self, index_path, index_name):
     if not whoosh.index.exists_in(index_path, index_name):
         print 'creating %s index at %s' % (index_name, index_path)
         if not os.path.exists(index_path):
             os.makedirs(index_path)
         schema = whoosh.fields.Schema(
                 id = whoosh.fields.ID(stored=True, unique=True),
                 artist = whoosh.fields.TEXT(stored=True),
                 title = whoosh.fields.TEXT(stored=True),
                 lyrics = whoosh.fields.TEXT(stored=True),
                 )
         index = whoosh.index.create_in(index_path, schema, index_name)
         index.writer().commit()
     return whoosh.index.open_dir(index_path, index_name)

def create_index_writer(index_path):
    '''
    Constructs a whoosh index writer, which has ID, artist and title fields

    :parameters:
        - index_path : str
            Path to whoosh index to be written

    :returns:
        - index : whoosh.writing.IndexWriter
            Whoosh index writer
    '''
    if not os.path.exists(index_path):
        os.mkdir(index_path)

    A = (whoosh.analysis.StandardAnalyzer(stoplist=None, minsize=1) |
         whoosh.analysis.CharsetFilter(accent_map))

    Schema = whoosh.fields.Schema(
        id=whoosh.fields.ID(stored=True),
        path=whoosh.fields.TEXT(stored=True),
        artist=whoosh.fields.TEXT(stored=True, analyzer=A),
        title=whoosh.fields.TEXT(stored=True, analyzer=A))

    index = whoosh.index.create_in(index_path, Schema)
    return index.writer()

def reindex_snippets():
    from flask_website.database import Snippet
    writer = index.writer()
    for snippet in Snippet.query.all():
        snippet.remove_from_search_index(writer)
        snippet.add_to_search_index(writer)
    writer.commit()

def update_documentation_index():
    from flask_website.docs import DocumentationPage
    writer = index.writer()
    for page in DocumentationPage.iter_pages():
        page.remove_from_search_index(writer)
        page.add_to_search_index(writer)
    writer.commit()

def update_model_based_indexes(session, flush_context):
    """Called by a session event, updates the model based documents."""
    to_delete = []
    to_add = []
    for model in session.new:
        if isinstance(model, Indexable):
            to_add.append(model)

    for model in session.dirty:
        if isinstance(model, Indexable):
            to_delete.append(model)
            to_add.append(model)

    for model in session.dirty:
        if isinstance(model, Indexable):
            to_delete.append(model)

    if not (to_delete or to_add):
        return

    writer = index.writer()
    for model in to_delete:
        model.remove_from_search_index(writer)
    for model in to_add:
        model.add_to_search_index(writer)
    writer.commit()

def _after_flush(app, changes):
    # Any db updates go through here. We check if any of these models have
    # ``__searchable__`` fields, indicating they need to be indexed. With these
    # we update the whoosh index for the model. If no index exists, it will be
    # created here; this could impose a penalty on the initial commit of a
    # model.
    if app.config.get('WHOOSH_DISABLED') is True:
        return
    bytype = {}  # sort changes by type so we can use per-model writer
    for change in changes:
        update = change[1] in ('update', 'insert')

        if hasattr(change[0].__class__, __searchable__):
            bytype.setdefault(change[0].__class__.__name__, []).append(
                (update, change[0]))
    if not bytype:
        return
    try:
        for model, values in bytype.items():
            index = whoosh_index(app, values[0][1].__class__)
            with index.writer() as writer:
                for update, v in values:
                    has_parent = isinstance(
                        v.__class__.__base__, DeclarativeMeta)
                    index_one_record(
                        v, not update, writer, index_parent=has_parent)
    except Exception as ex:
        logging.warning("FAIL updating index of %s msg: %s" % (model, str(ex)))

  def after_commit(self, session):
    """
    Any db updates go through here. We check if any of these models have
    ``__searchable__`` fields, indicating they need to be indexed. With these
    we update the whoosh index for the model. If no index exists, it will be
    created here; this could impose a penalty on the initial commit of a model.
    """

    for typ, values in self.to_update.iteritems():
      model_class = values[0][1].__class__
      index = self.index_for_model_class(model_class)
      with index.writer() as writer:
        primary_field = model_class.search_query.primary
        searchable = model_class.__searchable__

        for change_type, model in values:
          # delete everything. stuff that's updated or inserted will get
          # added as a new doc. Could probably replace this with a whoosh
          # update.

          if change_type == "deleted":
              writer.delete_by_term(primary_field, unicode(getattr(model, primary_field)))
          else:
            attrs = dict((key, getattr(model, key)) for key in searchable)
            attrs[primary_field] = unicode(getattr(model, primary_field))
            if change_type == "new":
                writer.add_document(**attrs)
            elif change_type == "changed":
                writer.update_document(**attrs)

    self.to_update = {}

def index_one_record(record, delete=False, writer=None, index_parent=False):
    index = whoosh_index(current_app, record.__class__)
    close = False
    if not writer:
        writer = index.writer()
        close = True
    if index_parent:
        # index parent class
        parent_writer = whoosh_index(
            current_app, record.__class__.__base__).writer()
    primary_field = record.pure_whoosh.primary_key_name
    searchable = index.schema.names()
    if not delete:
        attrs = {}
        for key in searchable:
            attrs[key] = unicode(getattr(record, key))
        attrs[primary_field] = unicode(
            getattr(record, primary_field))
        writer.update_document(**attrs)
        if index_parent:
            parent_writer.update_document(**attrs)
    else:
        writer.delete_by_term(
            primary_field, unicode(getattr(record, primary_field)))
        if index_parent:
            parent_writer.delete_by_term(
                primary_field, unicode(getattr(record, primary_field)))
    if close:
        writer.commit()

def _after_flush(app, changes):
    # Any db updates go through here. We check if any of these models have
    # ``__searchable__`` fields, indicating they need to be indexed. With these
    # we update the whoosh index for the model. If no index exists, it will be
    # created here; this could impose a penalty on the initial commit of a
    # model.

    bytype = {}  # sort changes by type so we can use per-model writer
    for change in changes:
        update = change[1] in ("update", "insert")

        if hasattr(change[0].__class__, __searchable__):
            bytype.setdefault(change[0].__class__.__name__, []).append((update, change[0]))

    for model, values in bytype.items():
        index = whoosh_index(app, values[0][1].__class__)
        with index.writer() as writer:
            primary_field = values[0][1].pure_whoosh.primary_key_name
            searchable = values[0][1].__searchable__

            for update, v in values:
                if update:
                    attrs = {}
                    for key in searchable:
                        try:
                            attrs[key] = unicode(getattr(v, key))
                        except AttributeError:
                            raise AttributeError("{0} does not have {1} field {2}".format(model, __searchable__, key))

                    attrs[primary_field] = unicode(getattr(v, primary_field))
                    writer.update_document(**attrs)
                else:
                    writer.delete_by_term(primary_field, unicode(getattr(v, primary_field)))

def create_index_writer(index_path):
    '''Create a new whoosh index in the given directory path.
    
    Input: directory in which to create the index
    
    Output: `whoosh.index` writer object
    '''
    

    if not os.path.exists(index_path):
        os.mkdir(index_path)

    analyzer = (whoosh.analysis.StemmingAnalyzer() | 
                whoosh.analysis.CharsetFilter(accent_map))

    schema = whoosh.fields.Schema(track_id=whoosh.fields.STORED,
                                  title=whoosh.fields.TEXT(stored=True, analyzer=analyzer),
                                  artist=whoosh.fields.TEXT(stored=True, analyzer=analyzer),
                                  album=whoosh.fields.TEXT(stored=True, analyzer=analyzer),
                                  collection=whoosh.fields.KEYWORD(stored=True),
                                  collection_id=whoosh.fields.NUMERIC(stored=True))

    index = whoosh.index.create_in(index_path, schema)

    return index.writer()

def _get_writer(index):
    writer = None
    while writer is None:
        try:
            writer = index.writer()
        except whoosh.index.LockError:
            time.sleep(0.25)

    return writer

 def rebuild_index_model(self, model_class, session):
   index = self.index_for_model_class(model_class)
   with index.writer() as writer:
     primary_field = model_class.search_query.primary
     searchable = model_class.__searchable__
     for i in session.query(model_class):
       attrs = dict((key, getattr(i, key)) for key in searchable)
       attrs[primary_field] = unicode(getattr(i, primary_field))
       writer.delete_by_term(primary_field, unicode(getattr(i, primary_field)))
       writer.add_document(**attrs)

def handle_postupdate(item_id): # pragma: no cover
    """Insert item data into indexer.

    """
    item = icecrate.items.by_item_id(item_id)

    writer = index.writer()
    writer.update_document(
        upc=item_id,
        name=item.get("name"),
        tags=list(icecrate.tags._split_tags(item.get("tags", ""))))
    writer.commit()

def createIndexWriter(indexPath):
    if not os.path.exists(indexPath):
        os.mkdir(indexPath)

    A = whoosh.analysis.FancyAnalyzer() | whoosh.analysis.CharsetFilter(accent_map)

    Schema = whoosh.fields.Schema(  song_id     = whoosh.fields.ID(stored=True),
                                    artist      = whoosh.fields.TEXT(stored=True, analyzer=A),
                                    title       = whoosh.fields.TEXT(stored=True, analyzer=A))

    index = whoosh.index.create_in(indexPath, Schema)
    return index.writer()
    pass