本文整理汇总了Python中utils.search.solr.Solr类的典型用法代码示例。如果您正苦于以下问题:Python Solr类的具体用法?Python Solr怎么用?Python Solr使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了Solr类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: get_user_tags
def get_user_tags(self, use_solr=True):
if use_solr:
query = SolrQuery()
query.set_dismax_query('')
filter_query = 'username:\"%s\"' % self.user.username
query.set_query_options(field_list=["id"], filter_query=filter_query)
query.add_facet_fields("tag")
query.set_facet_options("tag", limit=10, mincount=1)
solr = Solr(settings.SOLR_URL)
try:
results = SolrResponseInterpreter(solr.select(unicode(query)))
except SolrException as e:
return False
except Exception as e:
return False
return [{'name': tag, 'count': count} for tag, count in results.facets['tag']]
else:
return DelayedQueryExecuter("""
SELECT tags_tag.name AS name, X.c AS count
FROM ( SELECT tag_id, count(*) as c
FROM tags_taggeditem
LEFT JOIN sounds_sound ON object_id=sounds_sound.id
WHERE tags_taggeditem.user_id=%d AND
sounds_sound.moderation_state='OK' AND
sounds_sound.processing_state='OK'
GROUP BY tag_id
ORDER BY c
DESC LIMIT 10) AS X
LEFT JOIN tags_tag ON tags_tag.id=X.tag_id
ORDER BY tags_tag.name;""" % self.user_id)
示例2: add_sounds_to_solr
def add_sounds_to_solr(sounds):
solr = Solr(settings.SOLR_URL)
console_logger.info("creating XML")
documents = [convert_to_solr_document(s) for s in sounds]
console_logger.info("adding %d sounds to solr index" % len(documents))
console_logger.info("posting to Solr")
solr.add(documents)
示例3: tags
def tags(request, multiple_tags=None):
if multiple_tags:
multiple_tags = multiple_tags.split('/')
else:
multiple_tags = []
multiple_tags = sorted(filter(lambda x: x, multiple_tags))
try:
current_page = int(request.GET.get("page", 1))
except ValueError:
current_page = 1
solr = Solr(settings.SOLR_URL)
query = SolrQuery()
if multiple_tags:
query.set_query(" ".join("tag:\"" + tag + "\"" for tag in multiple_tags))
else:
query.set_query("*:*")
query.set_query_options(start=(current_page - 1) * settings.SOUNDS_PER_PAGE, rows=settings.SOUNDS_PER_PAGE, field_list=["id"], sort=["num_downloads desc"])
query.add_facet_fields("tag")
query.set_facet_options_default(limit=100, sort=True, mincount=1, count_missing=False)
query.set_group_field(group_field="grouping_pack")
query.set_group_options(group_func=None,
group_query=None,
group_rows=10,
group_start=0,
group_limit=1,
group_offset=0,
group_sort=None,
group_sort_ingroup=None,
group_format='grouped',
group_main=False,
group_num_groups=True,
group_cache_percent=0,
group_truncate=True) # Sets how many results from the same grup are taken into account for computing the facets
try:
results = SolrResponseInterpreter(solr.select(unicode(query)))
paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE)
num_results = paginator.count
non_grouped_number_of_results = results.non_grouped_number_of_matches
page = paginator.page(current_page)
error = False
tags = [dict(name=f[0], count=f[1]) for f in results.facets["tag"]]
docs = results.docs
resultids = [d.get("id") for d in docs]
resultsounds = sounds.models.Sound.objects.bulk_query_id(resultids)
allsounds = {}
for s in resultsounds:
allsounds[s.id] = s
for d in docs:
d["sound"] = allsounds[d["id"]]
except SolrException, e:
error = True
search_logger.error("SOLR ERROR - %s" % e)
示例4: perform_solr_query
def perform_solr_query(q, current_page):
"""
This util function performs the query to Solr and returns needed parameters to continue with the view.
The main reason to have this util function is to facilitate mocking in unit tests for this view.
"""
solr = Solr(settings.SOLR_URL)
results = SolrResponseInterpreter(solr.select(unicode(q)))
paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE)
page = paginator.page(current_page)
return results.non_grouped_number_of_matches, results.facets, paginator, page, results.docs
示例5: get_pack_tags
def get_pack_tags(pack_obj):
query = SolrQuery()
query.set_dismax_query('')
filter_query = 'username:\"%s\" pack:\"%s\"' % (pack_obj.user.username, pack_obj.name)
query.set_query_options(field_list=["id"], filter_query=filter_query)
query.add_facet_fields("tag")
query.set_facet_options("tag", limit=20, mincount=1)
try:
solr = Solr(settings.SOLR_URL)
results = SolrResponseInterpreter(solr.select(unicode(query)))
except (SolrException, Exception) as e:
# TODO: do something here?
return False
return results.facets
示例6: items
def items(self, obj):
if obj['query'] != "":
try:
solr = Solr(settings.SOLR_URL)
query = SolrQuery()
fields=[('id',4),
('tag', 3),
('description', 3),
('username', 2),
('pack_tokenized', 2),
('original_filename', 2),]
if obj['type'] == "phrase":
query.set_dismax_query('"' + obj['query'] + '"',query_fields=fields) # EXACT (not 100%)
elif obj['type'] == "any":
query.set_dismax_query(obj['query'],query_fields=[],minimum_match=0) # OR
else:
query.set_dismax_query(obj['query'],query_fields=[],minimum_match="100%") # AND
lim = obj['limit']
if lim > 100:
lim = 100
query.set_query_options(start=obj['offset'], rows=lim, filter_query="", sort=['created desc'])
try:
results = SolrResponseInterpreter(solr.select(unicode(query)))
sounds = []
for object in results.docs :
try:
sounds.append(object)
except: # This will happen if there are synchronization errors between solr index and the database. In that case sounds are ommited and both num_results and results per page might become inacurate
pass
logger.info("Sound pool search RSS")
return sounds
except SolrException, e:
return []
except:
return []
else:
return []
示例7: handle
def handle(self, *args, **options):
LIMIT = None
SLICE_SIZE = 500
solr_sound_ids = []
solr = Solr(url=settings.SOLR_URL)
query = SolrQuery()
query.set_dismax_query("") # Query to get ALL sounds
print "Retrieving ids from %i to %i"%(0,SLICE_SIZE)
query.set_query_options(field_list=["id"], rows = SLICE_SIZE, start = 0)
results = SolrResponseInterpreter(solr.select(unicode(query)))
solr_sound_ids += list_of_dicts_to_list_of_ids(results.docs)
total_num_documents = results.num_found
# Start iterating over other pages (slices)
if LIMIT:
number_of_documents = min(LIMIT,total_num_documents)
else:
number_of_documents = total_num_documents
for i in range(SLICE_SIZE,number_of_documents,SLICE_SIZE):
print "Retrieving ids from %i to %i"%(i,i+SLICE_SIZE)
query.set_query_options(field_list=["id"], rows = SLICE_SIZE, start = i)
results = SolrResponseInterpreter(solr.select(unicode(query)))
solr_sound_ids += list_of_dicts_to_list_of_ids(results.docs)
solr_sound_ids = sorted(list(set(solr_sound_ids)))
if LIMIT:
solr_sound_ids = solr_sound_ids[0:LIMIT]
print "%i document ids retrieved"%len(solr_sound_ids)
n_deleted = 0
print ""
for count,id in enumerate(solr_sound_ids):
sys.stdout.write("\rChecking doc %i of %i"%(count,len(solr_sound_ids)))
sys.stdout.flush()
if Sound.objects.filter(id=id,moderation_state="OK",processing_state="OK").exists():
pass
else:
# Sound does not exist in the Db or is not properly moderated and processed
print "\n\t - Deleting sound with id %i from solr index"%id
solr.delete_by_id(id)
n_deleted += 1
print "\n\nDONE! %i sounds deleted from solr index (it may take some minutes to actually see the changes in the page)"%n_deleted
示例8: get_all_sound_ids_from_solr
def get_all_sound_ids_from_solr(limit=False):
logger.info("getting all sound ids from solr.")
if not limit:
limit = 99999999999999
solr = Solr(settings.SOLR_URL)
solr_ids = []
solr_count = None
PAGE_SIZE = 2000
current_page = 1
while (len(solr_ids) < solr_count or solr_count is None) and len(solr_ids) < limit:
response = SolrResponseInterpreter(
solr.select(unicode(search_prepare_query(
'', '', search_prepare_sort('created asc', SEARCH_SORT_OPTIONS_WEB), current_page, PAGE_SIZE,
include_facets=False))))
solr_ids += [element['id'] for element in response.docs]
solr_count = response.num_found
current_page += 1
return sorted(solr_ids)
示例9: get_solr_results
def get_solr_results(search_form, page_size, max_pages, start_page=1, valid_ids=None, solr=None, offset=None):
if not solr:
solr = Solr(settings.SOLR_URL)
query_filter = search_form.cleaned_data['filter']
if valid_ids:
# Update solr filter to only return results in valid ids
ids_filter = 'id:(' + ' OR '.join([str(item) for item in valid_ids]) + ')'
if query_filter:
query_filter += ' %s' % ids_filter
else:
query_filter = ids_filter
solr_ids = []
solr_count = None
try:
current_page = start_page
n_page_requests = 1
# Iterate over solr result pages
while (len(solr_ids) < solr_count or solr_count == None) and n_page_requests <= max_pages:
query = search_prepare_query(unquote(search_form.cleaned_data['query'] or ""),
unquote(query_filter or ""),
search_form.cleaned_data['sort'],
current_page,
page_size,
grouping=False,
include_facets=False,
offset=offset)
result = SolrResponseInterpreter(solr.select(unicode(query)))
solr_ids += [element['id'] for element in result.docs]
solr_count = result.num_found
#print 'Solr page %i (total %i sounds)' % (current_page, solr_count)
current_page += 1
n_page_requests += 1
except SolrException as e:
raise ServerErrorException(msg='Search server error: %s' % e.message)
except Exception as e:
raise ServerErrorException(msg='The search server could not be reached or some unexpected error occurred.')
return solr_ids, solr_count
示例10: get_random_sound_from_solr
def get_random_sound_from_solr():
""" Get a random sound from solr.
This is used for random sound browsing. We filter explicit sounds,
but otherwise don't have any other restrictions on sound attributes
"""
solr = Solr(settings.SOLR_URL)
query = SolrQuery()
rand_key = random.randint(1, 10000000)
sort = ['random_%d asc' % rand_key]
filter_query = 'is_explicit:0'
query.set_query("*:*")
query.set_query_options(start=0, rows=1, field_list=["*"], filter_query=filter_query, sort=sort)
try:
response = SolrResponseInterpreter(solr.select(unicode(query)))
docs = response.docs
if docs:
return docs[0]
except (SolrException, socket.error):
pass
return {}
示例11: tags
def tags(request, multiple_tags=None):
if multiple_tags:
multiple_tags = multiple_tags.split('/')
else:
multiple_tags = []
multiple_tags = sorted(filter(lambda x:x, multiple_tags))
try:
current_page = int(request.GET.get("page", 1))
except ValueError:
current_page = 1
solr = Solr(settings.SOLR_URL)
query = SolrQuery()
if multiple_tags:
query.set_query(" ".join("tag:\"" + tag + "\"" for tag in multiple_tags))
else:
query.set_query("*:*")
query.set_query_options(start=(current_page - 1) * settings.SOUNDS_PER_PAGE, rows=settings.SOUNDS_PER_PAGE, field_list=["id"], sort=["num_downloads desc"])
query.add_facet_fields("tag")
query.set_facet_options_default(limit=100, sort=True, mincount=1, count_missing=False)
try:
results = SolrResponseInterpreter(solr.select(unicode(query)))
paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE)
page = paginator.page(current_page)
error = False
tags = [dict(name=f[0], count=f[1]) for f in results.facets["tag"]]
except SolrException, e:
error = True
search_logger.error("SOLR ERROR - %s" % e)
示例12: handle
def handle(self, *args, **options):
# init
solr = Solr(settings.SOLR_URL)
# Get all solr ids
print "Getting solr ids...",
solr_ids = get_all_sound_ids_from_solr()
print "done!"
# Get ell gaia ids
print "Getting gaia ids...",
gaia_ids = Similarity.get_all_sound_ids()
print "done!"
print "Getting freesound db data..."
# Get all moderated and processed sound ids
queryset = Sound.objects.filter(processing_state='OK', moderation_state='OK').order_by('id').only("id")
fs_mp = [sound.id for sound in queryset]
# Get ell moderated, processed and analysed sounds
queryset = Sound.objects.filter(processing_state='OK', moderation_state='OK', analysis_state='OK').order_by('id').only("id")
fs_mpa = [sound.id for sound in queryset]
print "done!"
print "\nNumber of sounds per index:\n--------------------------"
print "Solr index\t\t%i" % len(solr_ids)
print "Gaia index\t\t%i" % len(gaia_ids)
print "Freesound\t\t%i (moderated and processed)" % len(fs_mp)
print "Freesound\t\t%i (moderated, processed and analyzed)" % len(fs_mpa)
print "\n\n***************\nSOLR INDEX\n***************\n"
in_solr_not_in_fs = list(set(solr_ids).intersection(set(set(solr_ids).difference(fs_mp))))
in_fs_not_in_solr = list(set(fs_mp).intersection(set(set(fs_mp).difference(solr_ids))))
print "Sounds in solr but not in fs:\t%i" % len(in_solr_not_in_fs)
print "Sounds in fs but not in solr:\t%i" % len(in_fs_not_in_solr)
if not options['no-changes']:
# Mark fs sounds to go processing
if in_fs_not_in_solr:
print "Changing is_index_dirty_state of sounds that require it"
N = len(in_fs_not_in_solr)
for count, sid in enumerate(in_fs_not_in_solr):
sys.stdout.write('\r\tChanging state of sound sound %i of %i ' % (count+1, N))
sys.stdout.flush()
sound = Sound.objects.get(id=sid)
sound.set_single_field('is_index_dirty', True)
# Delete sounds from solr that are not in the db
if in_solr_not_in_fs:
print "\nDeleting sounds that should not be in solr"
N = len(in_solr_not_in_fs)
for count, sid in enumerate(in_solr_not_in_fs):
sys.stdout.write('\r\tDeleting sound %i of %i ' % (count+1, N))
sys.stdout.flush()
solr.delete_by_id(sid)
print "\n***************\nGAIA INDEX\n***************\n"
in_gaia_not_in_fs = list(set(gaia_ids).intersection(set(set(gaia_ids).difference(fs_mpa))))
in_fs_not_in_gaia = list(set(fs_mpa).intersection(set(set(fs_mpa).difference(gaia_ids))))
print "Sounds in gaia but not in fs:\t%i" % len(in_gaia_not_in_fs)
print "Sounds in fs but not in gaia:\t%i (only considering sounds correctly analyzed)" % len(in_fs_not_in_gaia)
#Similarity.save()
if not options['no-changes']:
# Mark fs sounds to go processing
if in_fs_not_in_gaia:
print "Changing similarity_state of sounds that require it"
N = len(in_fs_not_in_gaia)
for count, sid in enumerate(in_fs_not_in_gaia):
sys.stdout.write('\r\tChanging state of sound %i of %i ' % (count+1, N))
sys.stdout.flush()
sound = Sound.objects.get(id=sid)
sound.set_similarity_state('PE')
# Delete sounds from gaia that are not in the db
if in_gaia_not_in_fs:
print "\nDeleting sounds that should not be in gaia"
N = len(in_gaia_not_in_fs)
for count, sid in enumerate(in_gaia_not_in_fs):
sys.stdout.write('\r\tDeleting sound %i of %i ' % (count+1, N))
sys.stdout.flush()
Similarity.delete(sid)
示例13: add_sounds_to_solr
def add_sounds_to_solr(sounds):
solr = Solr(settings.SOLR_URL)
documents = [convert_to_solr_document(s) for s in sounds]
console_logger.info("Adding %d sounds to solr index" % len(documents))
logger.info("Adding %d sounds to solr index" % len(documents))
solr.add(documents)
示例14: search_forum
def search_forum(request):
search_query = request.GET.get("q", "")
filter_query = request.GET.get("f", "")
try:
current_page = int(request.GET.get("page", 1))
except ValueError:
current_page = 1
current_forum_name_slug = request.GET.get("forum", "").strip() # for context sensitive search
if current_forum_name_slug:
current_forum = get_object_or_404(forum.models.Forum.objects, name_slug=current_forum_name_slug)
else:
current_forum = None
sort = ["thread_created desc"]
# Parse advanced search options
advanced_search = request.GET.get("advanced_search", "")
date_from = request.GET.get("dt_from", "")
try:
df_parsed = datetime.datetime.strptime(date_from, "%Y-%m-%d")
date_from_display = df_parsed.strftime("%d-%m-%Y")
except ValueError:
date_from = ""
date_from_display = "Choose a Date"
date_to = request.GET.get("dt_to", "")
try:
dt_parsed = datetime.datetime.strptime(date_to, "%Y-%m-%d")
date_to_display = dt_parsed.strftime("%d-%m-%Y")
except ValueError:
date_to = ""
date_to_display = "Choose a Date"
if search_query.startswith("search in"):
search_query = ""
error = False
error_text = ""
paginator = None
num_results = None
page = None
results = []
if search_query.strip() != "" or filter_query:
# add current forum
if current_forum:
filter_query += "forum_name_slug:" + current_forum.name_slug
# add date range
if advanced_search == "1" and date_from != "" or date_to != "":
filter_query = __add_date_range(filter_query, date_from, date_to)
query = SolrQuery()
query.set_dismax_query(search_query, query_fields=[("thread_title", 4),
("post_body", 3),
("thread_author", 3),
("post_author", 3),
("forum_name", 2)])
query.set_highlighting_options_default(field_list=["post_body"],
fragment_size=200,
alternate_field="post_body", # TODO: revise this param
require_field_match=False,
pre="<strong>",
post="</strong>")
query.set_query_options(start=(current_page - 1) * settings.SOUNDS_PER_PAGE,
rows=settings.SOUNDS_PER_PAGE,
field_list=["id",
"forum_name",
"forum_name_slug",
"thread_id",
"thread_title",
"thread_author",
"thread_created",
"post_body",
"post_author",
"post_created",
"num_posts"],
filter_query=filter_query,
sort=sort)
query.set_group_field("thread_title_grouped")
query.set_group_options(group_limit=30)
solr = Solr(settings.SOLR_FORUM_URL)
try:
results = SolrResponseInterpreter(solr.select(unicode(query)))
paginator = SolrResponseInterpreterPaginator(results, settings.SOUNDS_PER_PAGE)
num_results = paginator.count
page = paginator.page(current_page)
error = False
except SolrException as e:
logger.warning("search error: query: %s error %s" % (query, e))
error = True
error_text = 'There was an error while searching, is your query correct?'
except Exception as e:
logger.error("Could probably not connect to Solr - %s" % e)
error = True
error_text = 'The search server could not be reached, please try again later.'
tvars = {
'advanced_search': advanced_search,
#.........这里部分代码省略.........
示例15: ServerErrorException
if e.status_code == 500:
raise ServerErrorException(msg=e.message, resource=resource)
elif e.status_code == 400:
raise BadRequestException(msg=e.message, resource=resource)
elif e.status_code == 404:
raise NotFoundException(msg=e.message, resource=resource)
else:
raise ServerErrorException(msg='Similarity server error: %s' % e.message, resource=resource)
except Exception, e:
raise ServerErrorException(msg='The similarity server could not be reached or some unexpected error occurred.', resource=resource)
elif not search_form.cleaned_data['descriptors_filter'] and not search_form.cleaned_data['target'] and not target_file:
# Standard text-based search
try:
solr = Solr(settings.SOLR_URL)
query = search_prepare_query(unquote(search_form.cleaned_data['query'] or ""),
unquote(search_form.cleaned_data['filter'] or ""),
search_form.cleaned_data['sort'],
search_form.cleaned_data['page'],
search_form.cleaned_data['page_size'],
grouping=search_form.cleaned_data['group_by_pack'],
include_facets=False)
result = SolrResponseInterpreter(solr.select(unicode(query)))
solr_ids = [element['id'] for element in result.docs]
solr_count = result.num_found
more_from_pack_data = None
if search_form.cleaned_data['group_by_pack']:
# If grouping option is on, store grouping info in a dictionary that we can add when serializing sounds