本文整理汇总了Python中whoosh.filedb.filestore.FileStorage.create_index方法的典型用法代码示例。如果您正苦于以下问题:Python FileStorage.create_index方法的具体用法?Python FileStorage.create_index怎么用?Python FileStorage.create_index使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类whoosh.filedb.filestore.FileStorage
的用法示例。
在下文中一共展示了FileStorage.create_index方法的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: build_index
# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def build_index(sa_session, whoosh_index_dir, path_to_repositories):
"""
Build the search indexes. One for repositories and another for tools within.
"""
# Rare race condition exists here and below
if not os.path.exists(whoosh_index_dir):
os.makedirs(whoosh_index_dir)
tool_index_dir = os.path.join(whoosh_index_dir, 'tools')
if not os.path.exists(tool_index_dir):
os.makedirs(tool_index_dir)
repo_index_storage = FileStorage(whoosh_index_dir)
tool_index_storage = FileStorage(tool_index_dir)
repo_index = repo_index_storage.create_index(repo_schema)
tool_index = tool_index_storage.create_index(tool_schema)
repo_index_writer = repo_index.writer()
tool_index_writer = tool_index.writer()
repos_indexed = 0
tools_indexed = 0
for repo in get_repos(sa_session, path_to_repositories):
repo_index_writer.add_document(id=repo.get('id'),
name=unicodify(repo.get('name')),
description=unicodify(repo.get('description')),
long_description=unicodify(repo.get('long_description')),
homepage_url=unicodify(repo.get('homepage_url')),
remote_repository_url=unicodify(repo.get('remote_repository_url')),
repo_owner_username=unicodify(repo.get('repo_owner_username')),
times_downloaded=repo.get('times_downloaded'),
approved=repo.get('approved'),
last_updated=repo.get('last_updated'),
full_last_updated=repo.get('full_last_updated'))
# Tools get their own index
for tool in repo.get('tools_list'):
tool_index_writer.add_document(id=unicodify(tool.get('id')),
name=unicodify(tool.get('name')),
version=unicodify(tool.get('version')),
description=unicodify(tool.get('description')),
help=unicodify(tool.get('help')),
repo_owner_username=unicodify(repo.get('repo_owner_username')),
repo_name=unicodify(repo.get('name')),
repo_id=repo.get('id'))
tools_indexed += 1
print(tools_indexed, 'tools (', tool.get('id'), ')')
repos_indexed += 1
print(repos_indexed, 'repos (', repo.get('id'), ')')
tool_index_writer.commit()
repo_index_writer.commit()
print("TOTAL repos indexed: ", repos_indexed)
print("TOTAL tools indexed: ", tools_indexed)
示例2: create_index
# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def create_index(sender=None, **kwargs):
"""Creates a File based whoosh index, location used is
settings.WHOOSH_INDEX so make sure that is set"""
if not os.path.exists(settings.WHOOSH_INDEX):
os.mkdir(settings.WHOOSH_INDEX)
storage = FileStorage(settings.WHOOSH_INDEX)
ix = storage.create_index(schema=WHOOSH_SCHEMA,
indexname="search")
示例3: _setup
# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def _setup(self, storage_directory):
schema = fields.Schema(
oid=fields.ID(stored=True, unique=True),
name=fields.ID())
schema.add('*', fields.TEXT, glob=True)
if storage_directory:
if os.path.exists(storage_directory):
self._using_existing_index = True
storage = FileStorage(storage_directory)
ix = storage.open_index()
else:
os.mkdir(storage_directory)
storage = FileStorage(storage_directory)
ix = storage.create_index(schema)
else:
storage = RamStorage()
ix = storage.create_index(schema)
return (schema, ix)
示例4: init_index
# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def init_index(index=".index"):
indexZ=index
if not os.path.exists(indexZ):
os.mkdir(indexZ) # os.rmdir(index)
storage = FileStorage(indexZ)
schema = Schema(name=TEXT(stored=True),ext=KEYWORD,title=TEXT(stored=True),content=TEXT,path=ID (stored=True),tags=KEYWORD)
ix = storage.create_index(schema)
ix = storage.open_index()
return ix
示例5: get_myindex
# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def get_myindex(indexdir='indexdir', filestore=False):
schema = get_schema()
if not filestore:
if not os.path.exists(indexdir):
os.mkdir(indexdir)
ix = index.create_in(indexdir, schema)
ix = index.open_dir(indexdir)
else:
storage = FileStorage(indexdir)
# TODO: When the indexdir has already exist
# the index object also use create_index,
# it should use open_dir as above method.
ix = storage.create_index(schema)
return ix
示例6: create_in
# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def create_in(dirname, schema, indexname=None):
"""Convenience function to create an index in a directory. Takes care of creating
a FileStorage object for you. indexname is t
:param dirname: the path string of the directory in which to create the index.
:param schema: a :class:`whoosh.fields.Schema` object describing the index's fields.
:param indexname: the name of the index to create; you only need to specify this if
you are creating multiple indexes within the same storage object.
:returns: :class:`Index`
"""
if not indexname:
indexname = _DEF_INDEX_NAME
from whoosh.filedb.filestore import FileStorage
storage = FileStorage(dirname)
return storage.create_index(schema, indexname)
示例7: build_index
# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def build_index( sa_session, whoosh_index_dir ):
storage = FileStorage( whoosh_index_dir )
index = storage.create_index( schema )
writer = index.writer()
def to_unicode( a_basestr ):
if type( a_basestr ) is str:
return unicode( a_basestr, 'utf-8' )
else:
return a_basestr
lddas_indexed = 0
for id, name, info, dbkey, message in get_lddas( sa_session ):
writer.add_document( id=id,
name=to_unicode( name ),
info=to_unicode( info ),
dbkey=to_unicode( dbkey ),
message=to_unicode( message ) )
lddas_indexed += 1
writer.commit()
print "Number of active library datasets indexed: ", lddas_indexed
示例8: newIndex
# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def newIndex():
'''
newIndex()
Creates the index/schema for the Whoosh module
INPUTS: (none)
OUTPUTS: idx -- index
'''
print '\tCreating a new Index in the current directory'
# Create an index to store the artist/title and lyrics
schm = Schema(Name=TEXT(stored=True), Ingr=KEYWORD(stored=True, commas=True))
# Create a directory called FAR_Storage; will contain the index
# See Whoosh documentation for more information
if not os.path.exists('FAR_Storage'):
os.mkdir('FAR_Storage')
idxDir ='FAR_Storage'
storage = FileStorage(idxDir)
idx = storage.create_index(schm, indexname='FAR')
idx = storage.open_index(indexname = 'FAR')
return idx
示例9: test_storage_creation
# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def test_storage_creation():
import tempfile, uuid
from whoosh import fields
from whoosh.filedb.filestore import FileStorage
schema = fields.Schema(text=fields.TEXT)
uid = uuid.uuid4()
dirpath = os.path.join(tempfile.gettempdir(), str(uid))
assert not os.path.exists(dirpath)
st = FileStorage(dirpath)
st.create()
assert os.path.exists(dirpath)
ix = st.create_index(schema)
with ix.writer() as w:
w.add_document(text=u("alfa bravo"))
w.add_document(text=u("bracho charlie"))
st.destroy()
assert not os.path.exists(dirpath)
示例10: build_index
# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def build_index( sa_session, toolshed_whoosh_index_dir ):
storage = FileStorage( toolshed_whoosh_index_dir )
index = storage.create_index( schema )
writer = index.writer()
def to_unicode( a_basestr ):
if type( a_basestr ) is str:
return unicode( a_basestr, 'utf-8' )
else:
return a_basestr
repos_indexed = 0
for ( id,
name,
description,
long_description,
homepage_url,
remote_repository_url,
repo_owner_username,
times_downloaded,
approved,
last_updated,
full_last_updated ) in get_repos( sa_session ):
writer.add_document( id = id,
name = to_unicode( name ),
description = to_unicode( description ),
long_description = to_unicode( long_description ),
homepage_url = to_unicode( homepage_url ),
remote_repository_url = to_unicode( remote_repository_url ),
repo_owner_username = to_unicode( repo_owner_username ),
times_downloaded = times_downloaded,
approved = approved,
last_updated = last_updated,
full_last_updated = full_last_updated )
repos_indexed += 1
writer.commit()
print "Number of repos indexed: ", repos_indexed
示例11: index
# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def index(self):
if self.empty:
if not os.path.exists(self.folder):
os.makedirs(self.folder)
st = FileStorage(self.folder)
ix = st.create_index(self.schema)
w = ix.writer()
w.add_document(name = u"beuha")
pipe = file.ID3Filter()
#[TODO] using itunes info for artwork?
cpt = 0
for track in pipe(ItunesParser(self.path)):
if track['album'] != None :
album = track['album'].encode('ascii', 'ignore')
else:
album = ""
#print track['artwork'], "[%s]" % album, track['name'].encode('ascii', 'ignore')
if cpt % 20 == 0:
print "\n%i " %cpt,
print '#',
#print track['album'], track['name']
w.add_document(
trackId = track['trackId'], name=track['name']
,artist=track['artist'], album=track['album'],
genre=track['genre'], location=track['location'],
artwork=boolean(track['artwork']),
trackNumber=track['trackNumber'], bitRate=track['bitRate']
)
#if cpt % 100 == 1:
# w.commit()
cpt += 1
print "\n\n%i tracks indexed" % cpt
w.commit()
ix.optimize()
ix.close()
else :
print "already indexed"
示例12: TinaIndex
# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
class TinaIndex():
"""
Open or Create a whoosh index
Provides searching methods
"""
def __init__( self, indexdir ):
self.writer = None
self.reader = None
self.searcher = None
self.indexdir = indexdir
self.storage = FileStorage(self.indexdir)
self.index = None
try:
self.index = self.storage.open_index()
except EmptyIndexError, e:
_logger.warning( "No existing index at %s : "%self.indexdir)
self.schema = TinaSchema()
if not os.path.exists(self.indexdir):
os.mkdir(self.indexdir)
self.index = self.storage.create_index(self.schema)
except LockError, le:
_logger.error("index LockError %s : "%self.indexdir)
raise LockError(le)
示例13: Library
# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
class Library(object):
RESULTS_LIMIT = 700
FUZZY_LIMIT = 5
SUGGESTIONS_LIMIT = 5
def __init__(self, dbsession, **settings):
"""Initializes Whoosh by setting up and loading indexes for lookup."""
self._dbsession = dbsession
self.schema = ChipSchema()
self.directory = settings.get(
'whoosh.store',
os.path.join(settings['config_path'], 'whoosh-data')
)
self.indexname = settings.get(
'whoosh.indexname',
'chips'
)
self.rebuild = asbool(settings.get('whoosh.rebuild', 'false'))
self.storage = FileStorage(self.directory)
self.setindex()
if self.rebuild:
self.setindex()
self.buildindex()
else:
self.setindex()
def setindex(self):
if self.rebuild and os.path.exists(self.directory):
shutil.rmtree(self.directory)
if not os.path.exists(self.directory):
os.mkdir(self.directory)
if whoosh.index.exists_in(
self.directory,
indexname=self.indexname
):
if self.rebuild:
shutil.rmtree(self.directory)
self.setindex()
else:
self.index = self.storage.open_index(indexname=self.indexname)
else:
self.index = self.storage.create_index(
self.schema,
indexname=self.indexname
)
def buildindex(self):
q = self._dbsession.query(Chip).all()
writer = self.index.writer()
for chip in q:
try:
version = chip.version.name
except AttributeError:
version = ''
writer.add_document(
id=str(chip.id),
indice=str(chip.indice),
indice_game=str(chip.indice_game),
name=chip.name.lower(),
name_jp=chip.name_jp,
name_display=chip.name,
game=chip.game.name.lower(),
game_enum=chip.game,
version=version,
version_enum=chip.version,
classification=chip.classification.name,
classification_enum=chip.classification,
element=chip.element.name,
element_enum=chip.element,
description=chip.description,
code=','.join(chip.codes_iter()).lower(),
size=str(chip.size),
damage_min=str(chip.damage_min),
damage_max=str(chip.damage_max),
recovery=str(chip.recovery),
rarity=str(chip.rarity)
)
writer.commit(writing.CLEAR)
def lookup(self, term, fuzzy=False, limit=None):
term = term.strip()
term = term.lower()
if limit:
limit = limit
else:
limit = self.RESULTS_LIMIT
fields = (
'indice',
'indice_game',
'name',
'name_jp',
'game',
#.........这里部分代码省略.........
示例14: setup_index
# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
def setup_index():
storage = FileStorage(data_dir('memory'))
storage.create()
return storage.create_index(TMSchema())
示例15: WhooshSearchBackend
# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import create_index [as 别名]
class WhooshSearchBackend(BaseSearchBackend):
# Word reserved by Whoosh for special use.
RESERVED_WORDS = (
'AND',
'NOT',
'OR',
'TO',
)
# Characters reserved by Whoosh for special use.
# The '\\' must come first, so as not to overwrite the other slash replacements.
RESERVED_CHARACTERS = (
'\\', '+', '-', '&&', '||', '!', '(', ')', '{', '}',
'[', ']', '^', '"', '~', '*', '?', ':', '.',
)
def __init__(self, connection_alias, **connection_options):
super(WhooshSearchBackend, self).__init__(connection_alias, **connection_options)
self.setup_complete = False
self.use_file_storage = True
self.post_limit = getattr(connection_options, 'POST_LIMIT', 128 * 1024 * 1024)
self.path = connection_options.get('PATH')
if connection_options.get('STORAGE', 'file') != 'file':
self.use_file_storage = False
if self.use_file_storage and not self.path:
raise ImproperlyConfigured("You must specify a 'PATH' in your settings for connection '%s'." % connection_alias)
def setup(self):
"""
Defers loading until needed.
"""
from haystack import connections
new_index = False
# Make sure the index is there.
if self.use_file_storage and not os.path.exists(self.path):
os.makedirs(self.path)
new_index = True
if self.use_file_storage and not os.access(self.path, os.W_OK):
raise IOError("The path to your Whoosh index '%s' is not writable for the current user/group." % self.path)
if self.use_file_storage:
self.storage = FileStorage(self.path)
else:
global LOCALS
if LOCALS.RAM_STORE is None:
LOCALS.RAM_STORE = RamStorage()
self.storage = LOCALS.RAM_STORE
self.content_field_name, self.schema = self.build_schema(connections[self.connection_alias].get_unified_index().all_searchfields())
self.parser = QueryParser(self.content_field_name, schema=self.schema)
if new_index is True:
self.index = self.storage.create_index(self.schema)
else:
try:
self.index = self.storage.open_index(schema=self.schema)
except index.EmptyIndexError:
self.index = self.storage.create_index(self.schema)
self.setup_complete = True
def build_schema(self, fields):
schema_fields = {
ID: WHOOSH_ID(stored=True, unique=True),
DJANGO_CT: WHOOSH_ID(stored=True),
DJANGO_ID: WHOOSH_ID(stored=True),
}
# Grab the number of keys that are hard-coded into Haystack.
# We'll use this to (possibly) fail slightly more gracefully later.
initial_key_count = len(schema_fields)
content_field_name = ''
for field_name, field_class in fields.items():
if field_class.is_multivalued:
if field_class.indexed is False:
schema_fields[field_class.index_fieldname] = IDLIST(stored=True, field_boost=field_class.boost)
else:
schema_fields[field_class.index_fieldname] = KEYWORD(stored=True, commas=True, scorable=True, field_boost=field_class.boost)
elif field_class.field_type in ['date', 'datetime']:
schema_fields[field_class.index_fieldname] = DATETIME(stored=field_class.stored)
elif field_class.field_type == 'integer':
schema_fields[field_class.index_fieldname] = NUMERIC(stored=field_class.stored, type=int, field_boost=field_class.boost)
elif field_class.field_type == 'float':
schema_fields[field_class.index_fieldname] = NUMERIC(stored=field_class.stored, type=float, field_boost=field_class.boost)
elif field_class.field_type == 'boolean':
# Field boost isn't supported on BOOLEAN as of 1.8.2.
schema_fields[field_class.index_fieldname] = BOOLEAN(stored=field_class.stored)
elif field_class.field_type == 'ngram':
schema_fields[field_class.index_fieldname] = NGRAM(minsize=3, maxsize=15, stored=field_class.stored, field_boost=field_class.boost)
elif field_class.field_type == 'edge_ngram':
schema_fields[field_class.index_fieldname] = NGRAMWORDS(minsize=2, maxsize=15, stored=field_class.stored, field_boost=field_class.boost)
else:
schema_fields[field_class.index_fieldname] = TEXT(stored=True, analyzer=StemmingAnalyzer(), field_boost=field_class.boost)
#.........这里部分代码省略.........