Python solr.SolrConnection类代码示例

 def _get_count(self):
     "Returns the total number of objects, across all pages."
     if self._count is None:
         solr = SolrConnection(settings.SOLR)  # TODO: maybe keep connection around?
         solr_response = solr.query(self._q, fields=['id'])
         self._count = int(solr_response.results.numFound)
     return self._count

def oralookup(pid=None, uuid=None, fields_to_return="f_name, f_subject, f_keyphrase, faculty, f_institution, thesis_type, content_type, collection", endpoint="http://ora.ouls.ox.ac.uk:8080/solr/select"):
  s = SolrConnection(endpoint)
  results = {}
  query = ""
  if pid:
    pid = "\:".join(pid.split(":"))
    query = "id:%s" % pid
  elif uuid:
    query = "id:uuid\:%s" % uuid
  else:
    return results
  # Running actual query (3 tries, failover)
  tries = 0
  while(tries != 3):
    try:
      r = s.query(q = query, fields = fields_to_return)
      logger.debug("Solr response: %s" % r.header)
      tries = 3
    except BadStatusLine:
      sleep(0.5)
      tries = tries + 1
  try:
    assert len(r.results) == 1
    return r.results[0]
  except ValueError:
    logger.warn("Couldn't parse json response from Solr endpoint: %s" % r)
    return {}
  except AssertionError:
    logger.warn("Couldn't assert that only a single result was fetched: %s" % results)
    return {}

def titlelookup(pid):
    s = SolrConnection("ora.ouls.ox.ac.uk:8080")
    results = {}
    query = ""
    if pid:
        pid = "\:".join(pid.split(":"))
        query = "id:%s" % pid
    elif uuid:
        query = "id:uuid\:%s" % uuid
    else:
        return results
    # Running actual query (3 tries, failover)
    tries = 0
    while tries != 3:
        try:
            r = s.search(q=query, wt="json", fl="title")
            logger.debug("Solr response: %s" % r)
            tries = 3
        except BadStatusLine:
            sleep(0.5)
            tries = tries + 1
    try:
        results = simplejson.loads(r)
        assert results["response"]["numFound"] == 1
        doc = results["response"]["docs"][0]
        return doc["title"]
    except ValueError:
        logger.warn("Couldn't parse json response from Solr endpoint: %s" % r)
        return {}
    except AssertionError:
        logger.warn("Couldn't assert that only a single result was fetched: %s" % results)
        return {}

def main():
    opts, args = parse_opts()
    logging.basicConfig(filename=opts.log_file, level=logging.DEBUG,
                        format='[%(asctime)s|%(levelname)s|%(name)s|%(threadName)s|%(message)s]')

    solr = SolrConnection(opts.solr_uri)
    protocol = LineProtocol()
    for request in protocol.input():
        try:
            query = build_query(request)
            if query is None:
                protocol.output(query_failed(), True)
                continue
            log.debug("Running query: " + str(query))
            results = solr.search(**query)
            if results is None:
                protocol.output({'code' : 400})
                continue
            resp = json.loads(results)
            ret = {
                'code' : 200,
                'json' : resp['response']
            }
            protocol.output(ret, True)
        except Exception:
            log.exception("Uncaught exception")
    return 0

def word_matches_for_page(page_id, words):
    """
    Gets a list of pre-analyzed words for a list of words on a particular
    page. So if you pass in 'manufacturer' you can get back a list like
    ['Manufacturer', 'manufacturers', 'MANUFACTURER'] etc ...
    """
    solr = SolrConnection(settings.SOLR)

    # Make sure page_id is of type str, else the following string
    # operation may result in a UnicodeDecodeError. For example, see
    # ticket #493
    if not isinstance(page_id, str):
        page_id = str(page_id)

    ocr_list = ['ocr', ]
    ocr_list.extend(['ocr_%s' % l for l in settings.SOLR_LANGUAGES])
    ocrs = ' OR '.join([query_join(words, o) for o in ocr_list])
    q = 'id:%s AND (%s)' % (page_id, ocrs)
    params = {"hl.snippets": 100, "hl.requireFieldMatch": 'true', "hl.maxAnalyzedChars": '102400'}
    response = solr.query(q, fields=['id'], highlight=ocr_list, **params)

    if page_id not in response.highlighting:
        return []

    words = set()
    for ocr in ocr_list:
        if ocr in response.highlighting[page_id]:
            for context in response.highlighting[page_id][ocr]:
                words.update(find_words(context))
    return list(words)

 def solr_delete(self):
     """
     Remove from solr index
     """
     solr_conn = SolrConnection(settings.SOLR_URL, persistent=False)
     solr_conn.delete_query('id:%s' % self.id)
     solr_conn.commit()

 def solr_index(self):
     """
     Write out to solr
     """
     solr_conn = SolrConnection(settings.SOLR_URL, persistent=False)
     solr_conn.add(**self.solr_doc)
     solr_conn.commit()

def index_evidence(evidence):
    evidence_medicine_list = []

    evidence_medicine = MedicineEvidenceSummary.objects.filter(evidence=evidence.id)
    for evimed in evidence_medicine: 
        if evimed.medicine.name not in evidence_medicine_list:
                evidence_medicine_list.append(evimed.medicine.name)

    # try to create a connection to a solr server and send medicine
    try:
        solr = SolrConnection(settings.SOLR_URL)
        solr.add(
            id = "evidence-%s-%s" % (evidence.language, evidence.id), 
            type = "evidence",
            title = evidence.title,            
            description = evidence.description,
            context = evidence.context,
            question = evidence.question,
            link = evidence.link,
            file = evidence.file,
            language = evidence.language,
            evidence_medicine = evidence_medicine_list,
        )
        response = solr.commit()
    except Exception as ex: 
        return False

    return True

def oralookup(
    pid=None,
    uuid=None,
    fields_to_return="f_name, f_subject, f_keyphrase, faculty, f_institution, thesis_type, content_type, collection",
):
    s = SolrConnection("ora.ouls.ox.ac.uk:8080")
    results = {}
    query = ""
    if pid:
        pid = "\:".join(pid.split(":"))
        query = "id:%s" % pid
    elif uuid:
        query = "id:uuid\:%s" % uuid
    else:
        return results
    # Running actual query (3 tries, failover)
    tries = 0
    while tries != 3:
        try:
            r = s.search(q=query, wt="json", fl=fields_to_return)
            logger.debug("Solr response: %s" % r)
            tries = 3
        except BadStatusLine:
            sleep(0.5)
            tries = tries + 1
    try:
        results = simplejson.loads(r)
        assert results["response"]["numFound"] == 1
        return results["response"]["docs"][0]
    except ValueError:
        logger.warn("Couldn't parse json response from Solr endpoint: %s" % r)
        return {}
    except AssertionError:
        logger.warn("Couldn't assert that only a single result was fetched: %s" % results)
        return {}

def index_title(title, solr=None):
    if solr is None:
        solr = SolrConnection(settings.SOLR)
    LOGGER.info("indexing title: lccn=%s", title.lccn)
    try:
        solr.add(**title.solr_doc)
    except Exception as e:
        LOGGER.exception(e)

def index_title(title, solr=None):
    if solr==None:
        solr = SolrConnection(settings.SOLR)
    _log.info("indexing title: lccn=%s" % title.lccn)
    try:
        solr.add(**title.solr_doc)
    except Exception, e:
        _log.exception(e)

def get_page_text(page):
    no_text = ["Text not available"]
    solr = SolrConnection(settings.SOLR)
    query = 'id:"%s"' % page.url
    solr_results = solr.query(query)
    results_attribute = getattr(solr_results, 'results', None)
    if isinstance(results_attribute, list) and len(results_attribute) > 0:
        return results_attribute[0].get('ocr', no_text)
    else:
        return no_text

def similar_pages(page):
    solr = SolrConnection(settings.SOLR)
    d = page.issue.date_issued
    year, month, day = '{0:02d}'.format(d.year), '{0:02d}'.format(d.month), '{0:02d}'.format(d.day) 
    date = ''.join(map(str, (year, month, day)))

    query = '+type:page AND date:%s AND %s AND NOT(lccn:%s)' % (date, query_join(map(lambda p: p.city, 
                                           page.issue.title.places.all()), 'city'), page.issue.title.lccn)
    response = solr.query(query, rows=25)
    results = response.results
    return map(lambda kwargs: utils.get_page(**kwargs), 
               map(lambda r: urlresolvers.resolve(r['id']).kwargs, results))

def execute_solr_query(query, fields, sort, sort_order, rows, start):
    solr = SolrConnection(settings.SOLR) # TODO: maybe keep connection around?
    solr_response = solr.query(query,
                               fields=['lccn', 'title',
                                       'edition',
                                       'place_of_publication',
                                       'start_year', 'end_year',
                                       'language'],
                               rows=rows,
                               sort=sort,
                               sort_order=sort_order,
                               start=start)
    return solr_response

def _refresh(field=None, data=None, path = None, isCron = None):
	from solr import SolrConnection
	from ID3 import *
	s = SolrConnection(SOLR_URL)
	if path and path != '*':
		#called by user		
		pathsArr = path.split(',')		
	else:
		#called from cron		
		pathsArr = folderpaths
	matches = []
	#handles modify, add
	#deletion will be handled in search when file in solr but not in path
	time.time()
	for path in pathsArr:
		for root, dirnames, filenames in os.walk(path):
			for extension in ['txt', 'log', 'py', 'pl', 'sql', 'mp3']:
				for filename in fnmatch.filter(filenames, '*.' + extension):				
					fullName = os.path.join(root, filename)
					if os.path.getsize(fullName) > 8800000:
						continue
					#print fullName
					if not isCron or (time.time() - os.path.getmtime(fullName) < 24*60*60):				
						try:
							#data = open(fullName, 'r').read().decode('raw_unicode_escape').replace('\n',' ').replace('\t',' ')
							if filename.endswith(('.txt', '.log', '.py', '.pl', '.sql')):								
								data = open(fullName, 'r').read()
								data = filterTxt(data)
							else:								
								audiofile = ID3(fullName)
								audiofilekeys = audiofile.keys()
								if 'TITLE' in audiofilekeys:
									data = audiofile['TITLE'] + " "
								if 'ARTIST' in audiofilekeys:
									data += audiofile['ARTIST'] + " "
								if 'ALBUM' in audiofilekeys:
									data += audiofile['ALBUM'] + " "
								if not data:
									data = ''
								data = data.strip()
							fullName = filterTxt(fullName)
							filename = filterTxt(filename)						
							s.add(id = fullName, name = filename, txt = data)
							s.commit()
						except:																	
							pass
							#print data
							#print traceback.format_exc()
							#print fullName	
							#sys.exit()					
						gc.collect()