本文整理汇总了Python中whoosh.filedb.filestore.FileStorage类的典型用法代码示例。如果您正苦于以下问题:Python FileStorage类的具体用法?Python FileStorage怎么用?Python FileStorage使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了FileStorage类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: update_index
def update_index(sender, instance, created, **kwargs):
storage = FileStorage(settings.WHOOSH_INDEX)
ix = storage.open_index()
try:
writer = ix.writer()
except:
return
tags = []
for t in instance.tags.all():
try:
tags.append(unicode(t.name))
except:
pass
tags = u','.join(tags)
try:
if created:
writer.add_document(title=instance.title, content=instance.content,tags=tags,author=instance.author.get_profile().name+u"\n"+instance.author.username,
id=unicode(instance.pk))
writer.commit()
else:
writer.update_document(title=instance.title, content=instance.content,tags=tags,author=instance.author.get_profile().name+u"\n"+instance.author.username,
id=unicode(instance.pk))
writer.commit()
except:
pass
示例2: update_index
def update_index(sender, instance, created, **kwargs):
if int(os.environ.get('SKIP_SEARCH_INDEX', '0')):
return
try:
url = unicode(instance.get_absolute_url())
except Exception:
log.critical('Cant resolve url. Content %r not indexed' % instance)
return
content = getattr(instance, 'content', None)
if content is None:
content = unicode(instance)
elif callable(content):
content = content()
storage = FileStorage(settings.WHOOSH_INDEX)
ix = storage.open_index(indexname='memopol')
writer = ix.writer()
if created:
writer.add_document(title=unicode(instance), content=content,
type=unicode(instance.__class__.__name__.lower()),
url=url)
writer.commit()
else:
writer.update_document(title=unicode(instance), content=content,
type=unicode(instance.__class__.__name__.lower()),
url=url)
writer.commit()
示例3: __init__
def __init__(self, path, masterkey=None):
FileStorage.__init__(self, path, supports_mmap=False)
self.masterkey = masterkey[:32]
self.signkey = masterkey[32:]
self._tmp_storage = self.temp_storage
self.length_cache = {}
self._open_files = {}
示例4: test_threaded_filelock
def test_threaded_filelock(self):
self.make_dir("testindex")
st = FileStorage("testindex")
lock1 = st.lock("testlock")
result = []
# The thread function tries to acquire the lock and
# then quits
def fn():
lock2 = st.lock("testlock")
gotit = try_for(lock2.acquire, 1.0, 0.1)
if gotit:
result.append(True)
lock2.release()
t = threading.Thread(target=fn)
# Acquire the lock in this thread
lock1.acquire()
# Start the other thread trying to acquire the lock
t.start()
# Wait for a bit
time.sleep(0.15)
# Release the lock
lock1.release()
# Wait for the other thread to finish
t.join()
# If the other thread got the lock, it should have
# appended something to the "results" list.
self.assertEqual(len(result), 1)
self.clean_file("testindex/testlock")
self.destroy_dir("testindex")
示例5: create_index
def create_index(sender=None, **kwargs):
"""Creates a File based whoosh index, location used is
settings.WHOOSH_INDEX so make sure that is set"""
if not os.path.exists(settings.WHOOSH_INDEX):
os.mkdir(settings.WHOOSH_INDEX)
storage = FileStorage(settings.WHOOSH_INDEX)
ix = storage.create_index(schema=WHOOSH_SCHEMA,
indexname="search")
示例6: get_index
def get_index():
try:
storage = FileStorage(settings.WHOOSH_INDEX)
return storage.open_index(indexname="search")
except IOError:
# No index? other error?
create_index()
storage = FileStorage(settings.WHOOSH_INDEX)
return storage.open_index(indexname="search")
示例7: init_index
def init_index(index=".index"):
indexZ=index
if not os.path.exists(indexZ):
os.mkdir(indexZ) # os.rmdir(index)
storage = FileStorage(indexZ)
schema = Schema(name=TEXT(stored=True),ext=KEYWORD,title=TEXT(stored=True),content=TEXT,path=ID (stored=True),tags=KEYWORD)
ix = storage.create_index(schema)
ix = storage.open_index()
return ix
示例8: build_index
def build_index(sa_session, whoosh_index_dir, path_to_repositories):
"""
Build the search indexes. One for repositories and another for tools within.
"""
# Rare race condition exists here and below
if not os.path.exists(whoosh_index_dir):
os.makedirs(whoosh_index_dir)
tool_index_dir = os.path.join(whoosh_index_dir, 'tools')
if not os.path.exists(tool_index_dir):
os.makedirs(tool_index_dir)
repo_index_storage = FileStorage(whoosh_index_dir)
tool_index_storage = FileStorage(tool_index_dir)
repo_index = repo_index_storage.create_index(repo_schema)
tool_index = tool_index_storage.create_index(tool_schema)
repo_index_writer = repo_index.writer()
tool_index_writer = tool_index.writer()
repos_indexed = 0
tools_indexed = 0
for repo in get_repos(sa_session, path_to_repositories):
repo_index_writer.add_document(id=repo.get('id'),
name=unicodify(repo.get('name')),
description=unicodify(repo.get('description')),
long_description=unicodify(repo.get('long_description')),
homepage_url=unicodify(repo.get('homepage_url')),
remote_repository_url=unicodify(repo.get('remote_repository_url')),
repo_owner_username=unicodify(repo.get('repo_owner_username')),
times_downloaded=repo.get('times_downloaded'),
approved=repo.get('approved'),
last_updated=repo.get('last_updated'),
full_last_updated=repo.get('full_last_updated'))
# Tools get their own index
for tool in repo.get('tools_list'):
tool_index_writer.add_document(id=unicodify(tool.get('id')),
name=unicodify(tool.get('name')),
version=unicodify(tool.get('version')),
description=unicodify(tool.get('description')),
help=unicodify(tool.get('help')),
repo_owner_username=unicodify(repo.get('repo_owner_username')),
repo_name=unicodify(repo.get('name')),
repo_id=repo.get('id'))
tools_indexed += 1
print(tools_indexed, 'tools (', tool.get('id'), ')')
repos_indexed += 1
print(repos_indexed, 'repos (', repo.get('id'), ')')
tool_index_writer.commit()
repo_index_writer.commit()
print("TOTAL repos indexed: ", repos_indexed)
print("TOTAL tools indexed: ", tools_indexed)
示例9: get_index
def get_index(index=".index"):
indexZ=index
if not os.path.exists(indexZ):
return "there is no index with this name %s!! use indexer to build the index" % index
sys.exit()
storage = FileStorage(indexZ)
ix = storage.open_index()
print "the index has %d docs" % ix.doc_count_all()
return ix
示例10: handle_noargs
def handle_noargs(self, **options):
# from settings import HAYSTACK_CONNECTIONS
# storage = FileStorage(HAYSTACK_CONNECTIONS['default']['PATH'])
storage = FileStorage('/dev/shm/whoosh/')
ix = storage.open_index('SPELL')
with ix.reader() as r:
for id in r.all_doc_ids():
print r.stored_fields(id)
示例11: _open_indexes
def _open_indexes(self):
"""open storage and open indexes"""
if not os.path.exists("index"):
os.mkdir("index")
storage = FileStorage("index")
# open or initialise index
if not storage.index_exists(indexname='MAIN'):
self.ix = storage.\
create_index(IndexerSchema, indexname='MAIN')
self.ix = storage.open_index(indexname='MAIN')
示例12: eval_get_ranked_set_baseline
def eval_get_ranked_set_baseline(self, basefile):
# Step 1: Read the saved keyterms for a subset of articles
# (created by analyze_baseline_queries)
g = Graph()
g.parse(self.generic_path("keyterms", "analyzed", ".n3"), format="n3")
articles = {}
for (s, p, o) in g:
if not str(s) in articles:
articles[str(s)] = []
articles[str(s)].append(str(o))
# Step 2: Open the large whoosh index containing the text of
# all cases. Then, create a query for each article based on
# the keyterms.
connector = query.Or
indexdir = os.path.sep.join([self.config.datadir, 'ecj', 'index'])
storage = FileStorage(indexdir)
idx = storage.open_index()
searcher = idx.searcher(weighting=scoring.BM25F())
res = {}
# for article in sorted(articles.keys()):
for article in self._articles(basefile):
terms = articles[article]
rankedset = []
#parser = qparser.QueryParser("content", idx.schema)
#q = parser.parse(connector.join(terms))
q = query.And([
# query.Term("articles", article),
connector([query.Term("content", x) for x in terms])
])
# print q
# self.log.debug("Article %s: %s", article, " or ".join(terms))
results = searcher.search(q, limit=None)
resultidx = 0
# self.log.info("Keyterms for result: %r" % results.key_terms("content", docs=10, numterms=10))
for result in results:
reslbl = "%s (%s)" % (
result['basefile'], results.score(resultidx))
rankedset.append([result['basefile'], reslbl])
# self.log.debug(u"\t%s: %2.2d" % (result['title'], results.score(resultidx)))
resultidx += 1
self.log.info("Created baseline ranked set for %s: Top result %s (of %s)" %
(article.split("/")[-1], rankedset[0][0], len(rankedset)))
# return just a list of URIs, no scoring information. But the
# full URI isnt available in the whoosh db, so we recreate it.
res[article] = ["http://lagen.nu/ext/celex/%s" % x[
0] for x in rankedset]
return res
示例13: update_index
def update_index(sender, instance, created, **kwargs):
storage = FileStorage(settings.WHOOSH_INDEX)
ix = storage.open_index(indexname="rarog")
writer = ix.writer()
if created:
writer.add_document(title=unicode(instance), body_html=instance.body_html,
url=unicode(instance.get_absolute_url()))
writer.commit()
else:
writer.update_document(title=unicode(instance), body_html=instance.body_html,
url=unicode(instance.get_absolute_url()))
writer.commit()
示例14: searchIndex
def searchIndex():
'''
searchindex()
Performs the requested search through the index/schema
INPUTS: idx -- desired index to search
OUTPUTS: results -- results of the search
'''
# Navigate to the LM index directory
c = ''
while True:
print 'The current directory is ' + os.getcwd()
ques = 'Is the LM index (directory) in the current directory? [y/n]\t'
c = raw_input(ques).lower()
if c == 'y' or c == 'yes':
idxDir = os.getcwd()
break
elif c == 'n' or c == 'no':
while True:
idxDir = raw_input('Where is it?\t').lower()
try:
os.chdir(idxDir)
break
except WindowsError:
print 'Sorry, I couldn\'t navigate to that directory'
break
elif c == 'q' or c == 'quit':
print '\tReturning to the Main Menu'
return
else:
print 'I\'m sorry, I don\'t understand what you mean. Try again.'
# Open the index
idxDir = idxDir + '/LM_Storage'
storage = FileStorage(idxDir)
idx = storage.open_index(indexname = 'LM')
# Determine what the user wants to search for
c = ''
while True:
ques = 'What would you like to search? song/artist [s], lyrics [L]\t'
c = raw_input(ques).lower()
if c == 's' or c == 'song/artist' or c == 'song':
searchForSong(idx)
break
elif c == 'l' or c == 'lyrics':
searchForLyrics(idx)
break
elif c == 'q' or c == 'quit':
print '\tReturning to the Main Menu'
return
else:
print 'I\'m sorry, I don\'t understand what you mean. Try again.'
示例15: search_does_exist
def search_does_exist(query):
#query = unicode(query, 'utf-8')
#query = unidecode(query)
storage = FileStorage("indexdir")
ix = storage.open_index(indexname="wiki")
from whoosh.qparser import QueryParser
with ix.searcher() as searcher:
query = QueryParser("title", ix.schema).parse(query)
whoosh_results = searcher.search(query, limit=1)
return len(whoosh_results) > 0