Python FileStorage.create_index方法代码示例

# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def build_index(sa_session, whoosh_index_dir, path_to_repositories):
    """
    Build the search indexes. One for repositories and another for tools within.
    """
    #  Rare race condition exists here and below
    if not os.path.exists(whoosh_index_dir):
        os.makedirs(whoosh_index_dir)
    tool_index_dir = os.path.join(whoosh_index_dir, 'tools')
    if not os.path.exists(tool_index_dir):
        os.makedirs(tool_index_dir)

    repo_index_storage = FileStorage(whoosh_index_dir)
    tool_index_storage = FileStorage(tool_index_dir)

    repo_index = repo_index_storage.create_index(repo_schema)
    tool_index = tool_index_storage.create_index(tool_schema)

    repo_index_writer = repo_index.writer()
    tool_index_writer = tool_index.writer()

    repos_indexed = 0
    tools_indexed = 0

    for repo in get_repos(sa_session, path_to_repositories):

        repo_index_writer.add_document(id=repo.get('id'),
                             name=unicodify(repo.get('name')),
                             description=unicodify(repo.get('description')),
                             long_description=unicodify(repo.get('long_description')),
                             homepage_url=unicodify(repo.get('homepage_url')),
                             remote_repository_url=unicodify(repo.get('remote_repository_url')),
                             repo_owner_username=unicodify(repo.get('repo_owner_username')),
                             times_downloaded=repo.get('times_downloaded'),
                             approved=repo.get('approved'),
                             last_updated=repo.get('last_updated'),
                             full_last_updated=repo.get('full_last_updated'))
        #  Tools get their own index
        for tool in repo.get('tools_list'):
            tool_index_writer.add_document(id=unicodify(tool.get('id')),
                                           name=unicodify(tool.get('name')),
                                           version=unicodify(tool.get('version')),
                                           description=unicodify(tool.get('description')),
                                           help=unicodify(tool.get('help')),
                                           repo_owner_username=unicodify(repo.get('repo_owner_username')),
                                           repo_name=unicodify(repo.get('name')),
                                           repo_id=repo.get('id'))
            tools_indexed += 1
            print(tools_indexed, 'tools (', tool.get('id'), ')')

        repos_indexed += 1
        print(repos_indexed, 'repos (', repo.get('id'), ')')

    tool_index_writer.commit()
    repo_index_writer.commit()

    print("TOTAL repos indexed: ", repos_indexed)
    print("TOTAL tools indexed: ", tools_indexed)

# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def create_index(sender=None, **kwargs):
    """Creates a File based whoosh index, location used is
    settings.WHOOSH_INDEX so make sure that is set"""
    if not os.path.exists(settings.WHOOSH_INDEX):
        os.mkdir(settings.WHOOSH_INDEX)
        storage = FileStorage(settings.WHOOSH_INDEX)
        ix = storage.create_index(schema=WHOOSH_SCHEMA,
                                  indexname="search")

# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
 def _setup(self, storage_directory):
     schema = fields.Schema(
         oid=fields.ID(stored=True, unique=True),
         name=fields.ID())
     schema.add('*', fields.TEXT, glob=True)
     if storage_directory:
         if  os.path.exists(storage_directory):
             self._using_existing_index = True
             storage = FileStorage(storage_directory)
             ix = storage.open_index()
         else:
             os.mkdir(storage_directory)
             storage = FileStorage(storage_directory)
             ix = storage.create_index(schema)
     else:
         storage = RamStorage()
         ix = storage.create_index(schema)
     return (schema, ix)

# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def init_index(index=".index"):
	indexZ=index
	if not os.path.exists(indexZ):
		os.mkdir(indexZ)      # os.rmdir(index)
	storage = FileStorage(indexZ)
	schema = Schema(name=TEXT(stored=True),ext=KEYWORD,title=TEXT(stored=True),content=TEXT,path=ID   (stored=True),tags=KEYWORD)
	ix = storage.create_index(schema)
	ix = storage.open_index()
	return ix

# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def get_myindex(indexdir='indexdir', filestore=False):
    schema = get_schema()
    if not filestore:
        if not os.path.exists(indexdir):
            os.mkdir(indexdir)
            ix = index.create_in(indexdir, schema)
        ix = index.open_dir(indexdir)
    else:
        storage = FileStorage(indexdir)
        # TODO: When the indexdir has already exist
        #       the index object also use create_index,
        #       it should use open_dir as above method.
        ix = storage.create_index(schema)
    return ix

# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def create_in(dirname, schema, indexname=None):
    """Convenience function to create an index in a directory. Takes care of creating
    a FileStorage object for you. indexname is t
    
    :param dirname: the path string of the directory in which to create the index.
    :param schema: a :class:`whoosh.fields.Schema` object describing the index's fields.
    :param indexname: the name of the index to create; you only need to specify this if
        you are creating multiple indexes within the same storage object.
    :returns: :class:`Index`
    """
    
    if not indexname:
        indexname = _DEF_INDEX_NAME
    
    from whoosh.filedb.filestore import FileStorage
    storage = FileStorage(dirname)
    return storage.create_index(schema, indexname)

# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def build_index( sa_session, whoosh_index_dir ):
    storage = FileStorage( whoosh_index_dir )
    index = storage.create_index( schema )
    writer = index.writer()
    def to_unicode( a_basestr ):
        if type( a_basestr ) is str:
            return unicode( a_basestr, 'utf-8' )
        else:
            return a_basestr
    lddas_indexed = 0
    for id, name, info, dbkey, message in get_lddas( sa_session ):
        writer.add_document( id=id,
                             name=to_unicode( name ),
                             info=to_unicode( info ),
                             dbkey=to_unicode( dbkey ),
                             message=to_unicode( message ) )
        lddas_indexed += 1
    writer.commit()
    print "Number of active library datasets indexed: ", lddas_indexed

# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def newIndex():
    '''
    newIndex()
    Creates the index/schema for the Whoosh module
    INPUTS: (none)
    OUTPUTS: idx -- index 
    '''
    print '\tCreating a new Index in the current directory'
    # Create an index to store the artist/title and lyrics
    schm = Schema(Name=TEXT(stored=True), Ingr=KEYWORD(stored=True, commas=True))
    # Create a directory called FAR_Storage; will contain the index
    # See Whoosh documentation for more information
    if not os.path.exists('FAR_Storage'):
        os.mkdir('FAR_Storage')
    idxDir ='FAR_Storage'
    storage = FileStorage(idxDir)
    idx = storage.create_index(schm, indexname='FAR')
    idx = storage.open_index(indexname = 'FAR')
    return idx

# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def test_storage_creation():
    import tempfile, uuid
    from whoosh import fields
    from whoosh.filedb.filestore import FileStorage

    schema = fields.Schema(text=fields.TEXT)
    uid = uuid.uuid4()
    dirpath = os.path.join(tempfile.gettempdir(), str(uid))
    assert not os.path.exists(dirpath)

    st = FileStorage(dirpath)
    st.create()
    assert os.path.exists(dirpath)

    ix = st.create_index(schema)
    with ix.writer() as w:
        w.add_document(text=u("alfa bravo"))
        w.add_document(text=u("bracho charlie"))

    st.destroy()
    assert not os.path.exists(dirpath)

# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def build_index( sa_session, toolshed_whoosh_index_dir ):
    storage = FileStorage( toolshed_whoosh_index_dir )
    index = storage.create_index( schema )
    writer = index.writer()
    def to_unicode( a_basestr ):
        if type( a_basestr ) is str:
            return unicode( a_basestr, 'utf-8' )
        else:
            return a_basestr

    repos_indexed = 0
    for ( id,
            name, 
            description, 
            long_description,
            homepage_url,
            remote_repository_url,
            repo_owner_username,
            times_downloaded,
            approved,
            last_updated,
            full_last_updated ) in get_repos( sa_session ):

        writer.add_document( id = id,
                             name = to_unicode( name ),
                             description = to_unicode( description ), 
                             long_description = to_unicode( long_description ), 
                             homepage_url = to_unicode( homepage_url ), 
                             remote_repository_url = to_unicode( remote_repository_url ), 
                             repo_owner_username = to_unicode( repo_owner_username ),
                             times_downloaded = times_downloaded,
                             approved = approved,
                             last_updated = last_updated,
                             full_last_updated = full_last_updated )
        repos_indexed += 1
    writer.commit()
    print "Number of repos indexed: ", repos_indexed

# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
	def index(self):
		if self.empty:
			if not os.path.exists(self.folder):
				os.makedirs(self.folder)
			st = FileStorage(self.folder)
			ix = st.create_index(self.schema)
			w = ix.writer()
			w.add_document(name = u"beuha")
			pipe = file.ID3Filter()
			#[TODO] using itunes info for artwork?
			cpt = 0
			for track in pipe(ItunesParser(self.path)):
				if track['album'] != None : 
					album = track['album'].encode('ascii', 'ignore')
				else:
					album = ""
				#print track['artwork'], "[%s]" % album, track['name'].encode('ascii', 'ignore')
				if cpt % 20 == 0:
					print "\n%i " %cpt,
				print '#',
				#print track['album'], track['name']
				w.add_document(
					trackId = track['trackId'], name=track['name']
					,artist=track['artist'], album=track['album'],
					genre=track['genre'], location=track['location'],
					artwork=boolean(track['artwork']),
					trackNumber=track['trackNumber'], bitRate=track['bitRate']
				)
				#if cpt % 100 == 1:
				#	w.commit()
				cpt += 1
			print "\n\n%i tracks indexed" % cpt
			w.commit()
			ix.optimize()
			ix.close()
		else :
			print "already indexed"

# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
class TinaIndex():
    """
    Open or Create a whoosh index
    Provides searching methods
    """

    def __init__( self, indexdir ):
        self.writer = None
        self.reader = None
        self.searcher = None
        self.indexdir = indexdir
        self.storage = FileStorage(self.indexdir)
        self.index = None
        try:
            self.index = self.storage.open_index()
        except EmptyIndexError, e:
            _logger.warning( "No existing index at %s : "%self.indexdir)
            self.schema = TinaSchema()
            if not os.path.exists(self.indexdir):
                os.mkdir(self.indexdir)
            self.index = self.storage.create_index(self.schema)
        except LockError, le:
            _logger.error("index LockError %s : "%self.indexdir)
            raise LockError(le)

# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
class Library(object):

    RESULTS_LIMIT = 700
    FUZZY_LIMIT = 5
    SUGGESTIONS_LIMIT = 5
    
    def __init__(self, dbsession, **settings):
        """Initializes Whoosh by setting up and loading indexes for lookup."""
        self._dbsession = dbsession
        self.schema = ChipSchema()
        self.directory = settings.get(
            'whoosh.store',
            os.path.join(settings['config_path'], 'whoosh-data')
        )
        self.indexname = settings.get(
            'whoosh.indexname',
            'chips'
        )
        self.rebuild = asbool(settings.get('whoosh.rebuild', 'false'))
        self.storage = FileStorage(self.directory)

        self.setindex()
        
        if self.rebuild:
            self.setindex()
            self.buildindex()
        else:
            self.setindex()

    def setindex(self):
        if self.rebuild and os.path.exists(self.directory):
            shutil.rmtree(self.directory)

        if not os.path.exists(self.directory):
            os.mkdir(self.directory)

        if whoosh.index.exists_in(
            self.directory,
            indexname=self.indexname
        ):
            if self.rebuild:
                shutil.rmtree(self.directory)
                self.setindex()
            else:
                self.index = self.storage.open_index(indexname=self.indexname)
        else:
            self.index = self.storage.create_index(
                self.schema,
                indexname=self.indexname
            )
            
    def buildindex(self):
        q = self._dbsession.query(Chip).all()
        writer = self.index.writer()
        for chip in q:
            try:
                version = chip.version.name
            except AttributeError:
                version = ''

            writer.add_document(
                id=str(chip.id),
                indice=str(chip.indice),
                indice_game=str(chip.indice_game),
                name=chip.name.lower(),
                name_jp=chip.name_jp,
                name_display=chip.name,
                game=chip.game.name.lower(),
                game_enum=chip.game,
                version=version,
                version_enum=chip.version,
                classification=chip.classification.name,
                classification_enum=chip.classification,
                element=chip.element.name,
                element_enum=chip.element,
                description=chip.description,
                code=','.join(chip.codes_iter()).lower(),
                size=str(chip.size),
                damage_min=str(chip.damage_min),
                damage_max=str(chip.damage_max),
                recovery=str(chip.recovery),
                rarity=str(chip.rarity)
            )
        writer.commit(writing.CLEAR)
        
    def lookup(self, term, fuzzy=False, limit=None):
        term = term.strip()
        term = term.lower()

        if limit:
            limit = limit
        else:
            limit = self.RESULTS_LIMIT

        fields = (
            'indice',
            'indice_game',
            'name',
            'name_jp',
            'game',
#.........这里部分代码省略.........

# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def setup_index():
    storage = FileStorage(data_dir('memory'))
    storage.create()
    return storage.create_index(TMSchema())

# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
class WhooshSearchBackend(BaseSearchBackend):
    # Word reserved by Whoosh for special use.
    RESERVED_WORDS = (
        'AND',
        'NOT',
        'OR',
        'TO',
    )
    
    # Characters reserved by Whoosh for special use.
    # The '\\' must come first, so as not to overwrite the other slash replacements.
    RESERVED_CHARACTERS = (
        '\\', '+', '-', '&&', '||', '!', '(', ')', '{', '}',
        '[', ']', '^', '"', '~', '*', '?', ':', '.',
    )
    
    def __init__(self, connection_alias, **connection_options):
        super(WhooshSearchBackend, self).__init__(connection_alias, **connection_options)
        self.setup_complete = False
        self.use_file_storage = True
        self.post_limit = getattr(connection_options, 'POST_LIMIT', 128 * 1024 * 1024)
        self.path = connection_options.get('PATH')
        
        if connection_options.get('STORAGE', 'file') != 'file':
            self.use_file_storage = False
        
        if self.use_file_storage and not self.path:
            raise ImproperlyConfigured("You must specify a 'PATH' in your settings for connection '%s'." % connection_alias)
    
    def setup(self):
        """
        Defers loading until needed.
        """
        from haystack import connections
        new_index = False
        
        # Make sure the index is there.
        if self.use_file_storage and not os.path.exists(self.path):
            os.makedirs(self.path)
            new_index = True
        
        if self.use_file_storage and not os.access(self.path, os.W_OK):
            raise IOError("The path to your Whoosh index '%s' is not writable for the current user/group." % self.path)
        
        if self.use_file_storage:
            self.storage = FileStorage(self.path)
        else:
            global LOCALS
            
            if LOCALS.RAM_STORE is None:
                LOCALS.RAM_STORE = RamStorage()
            
            self.storage = LOCALS.RAM_STORE
        
        self.content_field_name, self.schema = self.build_schema(connections[self.connection_alias].get_unified_index().all_searchfields())
        self.parser = QueryParser(self.content_field_name, schema=self.schema)
        
        if new_index is True:
            self.index = self.storage.create_index(self.schema)
        else:
            try:
                self.index = self.storage.open_index(schema=self.schema)
            except index.EmptyIndexError:
                self.index = self.storage.create_index(self.schema)
        
        self.setup_complete = True
    
    def build_schema(self, fields):
        schema_fields = {
            ID: WHOOSH_ID(stored=True, unique=True),
            DJANGO_CT: WHOOSH_ID(stored=True),
            DJANGO_ID: WHOOSH_ID(stored=True),
        }
        # Grab the number of keys that are hard-coded into Haystack.
        # We'll use this to (possibly) fail slightly more gracefully later.
        initial_key_count = len(schema_fields)
        content_field_name = ''
        
        for field_name, field_class in fields.items():
            if field_class.is_multivalued:
                if field_class.indexed is False:
                    schema_fields[field_class.index_fieldname] = IDLIST(stored=True, field_boost=field_class.boost)
                else:
                    schema_fields[field_class.index_fieldname] = KEYWORD(stored=True, commas=True, scorable=True, field_boost=field_class.boost)
            elif field_class.field_type in ['date', 'datetime']:
                schema_fields[field_class.index_fieldname] = DATETIME(stored=field_class.stored)
            elif field_class.field_type == 'integer':
                schema_fields[field_class.index_fieldname] = NUMERIC(stored=field_class.stored, type=int, field_boost=field_class.boost)
            elif field_class.field_type == 'float':
                schema_fields[field_class.index_fieldname] = NUMERIC(stored=field_class.stored, type=float, field_boost=field_class.boost)
            elif field_class.field_type == 'boolean':
                # Field boost isn't supported on BOOLEAN as of 1.8.2.
                schema_fields[field_class.index_fieldname] = BOOLEAN(stored=field_class.stored)
            elif field_class.field_type == 'ngram':
                schema_fields[field_class.index_fieldname] = NGRAM(minsize=3, maxsize=15, stored=field_class.stored, field_boost=field_class.boost)
            elif field_class.field_type == 'edge_ngram':
                schema_fields[field_class.index_fieldname] = NGRAMWORDS(minsize=2, maxsize=15, stored=field_class.stored, field_boost=field_class.boost)
            else:
                schema_fields[field_class.index_fieldname] = TEXT(stored=True, analyzer=StemmingAnalyzer(), field_boost=field_class.boost)
            
#.........这里部分代码省略.........