Python SolrConnection.add方法代码示例

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import add [as 别名]
 def solr_index(self):
     """
     Write out to solr
     """
     solr_conn = SolrConnection(settings.SOLR_URL, persistent=False)
     solr_conn.add(**self.solr_doc)
     solr_conn.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import add [as 别名]
def index_evidence(evidence):
    evidence_medicine_list = []

    evidence_medicine = MedicineEvidenceSummary.objects.filter(evidence=evidence.id)
    for evimed in evidence_medicine: 
        if evimed.medicine.name not in evidence_medicine_list:
                evidence_medicine_list.append(evimed.medicine.name)

    # try to create a connection to a solr server and send medicine
    try:
        solr = SolrConnection(settings.SOLR_URL)
        solr.add(
            id = "evidence-%s-%s" % (evidence.language, evidence.id), 
            type = "evidence",
            title = evidence.title,            
            description = evidence.description,
            context = evidence.context,
            question = evidence.question,
            link = evidence.link,
            file = evidence.file,
            language = evidence.language,
            evidence_medicine = evidence_medicine_list,
        )
        response = solr.commit()
    except Exception as ex: 
        return False

    return True

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import add [as 别名]
def index_title(title, solr=None):
    if solr is None:
        solr = SolrConnection(settings.SOLR)
    LOGGER.info("indexing title: lccn=%s", title.lccn)
    try:
        solr.add(**title.solr_doc)
    except Exception as e:
        LOGGER.exception(e)

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import add [as 别名]
def index_title(title, solr=None):
    if solr==None:
        solr = SolrConnection(settings.SOLR)
    _log.info("indexing title: lccn=%s" % title.lccn)
    try:
        solr.add(**title.solr_doc)
    except Exception, e:
        _log.exception(e)

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import add [as 别名]
def _refresh(field=None, data=None, path = None, isCron = None):
	from solr import SolrConnection
	from ID3 import *
	s = SolrConnection(SOLR_URL)
	if path and path != '*':
		#called by user		
		pathsArr = path.split(',')		
	else:
		#called from cron		
		pathsArr = folderpaths
	matches = []
	#handles modify, add
	#deletion will be handled in search when file in solr but not in path
	time.time()
	for path in pathsArr:
		for root, dirnames, filenames in os.walk(path):
			for extension in ['txt', 'log', 'py', 'pl', 'sql', 'mp3']:
				for filename in fnmatch.filter(filenames, '*.' + extension):				
					fullName = os.path.join(root, filename)
					if os.path.getsize(fullName) > 8800000:
						continue
					#print fullName
					if not isCron or (time.time() - os.path.getmtime(fullName) < 24*60*60):				
						try:
							#data = open(fullName, 'r').read().decode('raw_unicode_escape').replace('\n',' ').replace('\t',' ')
							if filename.endswith(('.txt', '.log', '.py', '.pl', '.sql')):								
								data = open(fullName, 'r').read()
								data = filterTxt(data)
							else:								
								audiofile = ID3(fullName)
								audiofilekeys = audiofile.keys()
								if 'TITLE' in audiofilekeys:
									data = audiofile['TITLE'] + " "
								if 'ARTIST' in audiofilekeys:
									data += audiofile['ARTIST'] + " "
								if 'ALBUM' in audiofilekeys:
									data += audiofile['ALBUM'] + " "
								if not data:
									data = ''
								data = data.strip()
							fullName = filterTxt(fullName)
							filename = filterTxt(filename)						
							s.add(id = fullName, name = filename, txt = data)
							s.commit()
						except:																	
							pass
							#print data
							#print traceback.format_exc()
							#print fullName	
							#sys.exit()					
						gc.collect()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import add [as 别名]
def index_missing_pages():
    """
    index all pages that are missing from solr in the database
    """
    solr = SolrConnection(settings.SOLR)
    count = 0
    pages = models.Page.objects.filter(indexed=False).all()
    number_of_pages = len(pages)
    for page in pages:
        LOGGER.info("[%s of %s] indexing page: %s", count, number_of_pages, page.url)
        solr.add(**page.solr_doc)
        count += 1
        page.indexed = True
        page.save()
    solr.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import add [as 别名]
def build_index(**kwargs):
    """
    gets product/sku information from cps DB and indexes them in solr

    existing solr index is wiped before indexing. Revisit if this strategy
    does not work
    """
    # index status log message granularity
    log_index_status_chunks = 25000
    solr = SolrConnection(settings.SOLR)
    clear_index(solr=solr)
    count = 0
    fieldnames = (
        "name",
        "id",
        "description",
        "long_description",
        "age",
        "gender",
        "brand",
        "str_brand",
        "merchant",
        "str_merchant",
        "category",
        "str_category",
        "price",
        "sale_price",
        "buy_url",
        "image",
    )
    start = datetime.now()
    log.info("Reading product info from the database.....")
    products = db.get_cps_data()
    log.info("Building SOLR index.....")
    for product in products:
        try:
            product_record = dict(zip(fieldnames, product))
            solr.add(**product_record)
            count += 1
        except Exception, e:
            log.exception(e)
            continue
        if count % log_index_status_chunks == 0:
            log.info("Indexed %d products in %s" % (count, datetime.now() - start))

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import add [as 别名]
def index_pages():
    """index all the pages that are modeled in the database
    """
    solr = SolrConnection(settings.SOLR)
    solr.delete_query('type:page')
    cursor = connection.cursor()
    cursor.execute("SELECT id FROM core_page")
    count = 0
    while True:
        row = cursor.fetchone()
        if row is None:
            break
        page = models.Page.objects.get(id=row[0])
        LOGGER.info("[%s] indexing page: %s", count, page.url)
        solr.add(**page.solr_doc)
        count += 1
        if count % 100 == 0:
            reset_queries()
    solr.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import add [as 别名]
        "uuid": [u"78755d851f9a453b84a51b1c00c68553"],
        "depositor": "zool0982"
        #               'identifier': ['fri_day1'],
        #                'aggregatedResource': ['http://datafinder-d2v.bodleian.ox.ac.uk/DataFinder/datasets/fri_day1/df_manifest.rdf'],
        #                'mediator': ['admin'],
        #               'text': ['', 'http://vocab.ox.ac.uk/projectfunding#', '', 'seeking_approval', '', '', 'yes', '', ''],
        #               'depositor': ['zool0982'],
        #               'embargoedUntilDate': ['2083-06-21T14:08:45Z'],
        #               'alternative': ['fri_day1'],
        #               'subject': [''],
        #               'rights': ['http://ora.ouls.ox.ac.uk/objects/uuid%3A1d00eebb-8fed-46ad-8e38-45dbdb4b224c'],
        #               'publisher': ['Bodleian Libraries, University of Oxford'],
        #               'license': ['CC0 1.0 Universal (CC0 1.0). See http://creativecommons.org/publicdomain/zero/1.0/legalcode'],
        #               'language': [''],
        #               'title': ['fri_day1'],
        #               'embargoStatus': ['True'],
        #               'description': [''],
        #               'format': [''],
        #               'modified': ['2013-06-21 14:08:45.525602'],
        #               'currentVersion': ['2'],
        #               'created': ['2013-06-21 14:08:45.253033'],
        #               'issued': [''],
        #               'type': ['', 'http://vocab.ox.ac.uk/dataset/schema#DataSet']
    }

    #   solr_doc = {'identifier': ['fri_day1'], 'aggregatedResource': ['http://datafinder-d2v.bodleian.ox.ac.uk/DataFinder/datasets/fri_day1/df_manifest.rdf'], 'mediator': ['admin'], 'text': ['', '', '', 'http://vocab.ox.ac.uk/projectfunding#', '', 'yes', '', 'seeking_approval', ''], 'depositor': ['zool0982'], 'embargoedUntilDate': ['2083-06-21T14:08:45Z'], 'alternative': ['fri_day1'], 'subject': [''], 'rights': ['http://ora.ouls.ox.ac.uk/objects/uuid%3A1d00eebb-8fed-46ad-8e38-45dbdb4b224c'], 'publisher': ['', 'Bodleian Libraries, University of Oxford'], 'license': ['CC0 1.0 Universal (CC0 1.0). See http://creativecommons.org/publicdomain/zero/1.0/legalcode'], 'uuid': [u'4fb84512bfaf4927945ea3c241bf21c0'], 'language': [''], 'title': ['fri_day1'], 'embargoStatus': ['True'], 'description': [''], 'format': [''], 'modified': ['2013-06-21 14:08:45.525602'], 'id': ['fri_day1'], 'currentVersion': ['2'], 'created': ['2013-06-21 14:08:45.253033'], 'issued': [''], 'silo': ['DataFinder'], 'type': ['http://vocab.ox.ac.uk/dataset/schema#DataSet', '']}
    # solr_doc = {'identifier': ['mond_ay2'], 'aggregatedResource': ['http://datafinder-d2v.bodleian.ox.ac.uk/DataFinder/datasets/mond_ay2/df_manifest.rdf'], 'mediator': ['admin'], 'text': ['', 'http://vocab.ox.ac.uk/projectfunding#', '', 'seeking_approval', '', 'yes', '', '', ''], 'depositor': 'zool0982', 'alternative': ['mond_ay2'], 'embargoedUntilDate': ['2083-06-24T03:41:53Z'], 'subject': [''], 'rights': ['http://ora.ouls.ox.ac.uk/objects/uuid%3A1d00eebb-8fed-46ad-8e38-45dbdb4b224c'], 'publisher': ['Bodleian Libraries, University of Oxford'], 'license': ['CC0 1.0 Universal (CC0 1.0). See http://creativecommons.org/publicdomain/zero/1.0/legalcode'], 'uuid': [u'78755d851f9a453b84a51b1c00c68553'], 'language': [''], 'title': ['mond_ay2'], 'embargoStatus': ['True'], 'description': ['mond_ay2'], 'format': [''], 'modified': ['2013-06-24 03:41:53.988847'], 'id': ['mond_ay2'], 'currentVersion': ['2'], 'created': ['2013-06-24 03:41:53.618090'], 'issued': [''], 'silo': ['DataFinder'], 'type': ['', 'http://vocab.ox.ac.uk/dataset/schema#DataSet']}
    # print repr(solr_doc)
    solr.add(_commit=True, **solr_doc)
    solr.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import add [as 别名]
     # Get rdf graph from manifest for dataset
     graph = None
     tries = 0
     while tries < 5:
         response = db.getFile(silo_name, itemid, "manifest.rdf")
         if db.good(response):
             manifest = response.results
             graph = ConjunctiveGraph()
             graph.parse(StringIO(manifest), "xml")
             break
         else:
             tries += 1
     if state_info and graph:
         solr_doc = gather_document(silo_name, itemid, graph, state_info, debug=True)
         try:
             solr.add(_commit=False, **solr_doc)
         except Exception, e:
             logger.error("Error adding document to solr id:%s in silo:%s\n" % (itemid, silo_name))
             try:
                 logger.error("%s\n\n" % str(e))
             except:
                 pass
             rq.task_failed()
             continue
     else:
         logger.error("Error gathering state and manifest info for id:%s in silo:%s\n" % (itemid, silo_name))
         rq.task_failed()
         continue
 else:
     solr_doc = {"id": silo_name, "silo": silo_name, "type": "Silo", "uuid": silo_name}
     # Get state infor for silo

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import add [as 别名]
def solr_index(med):
    lists = []
    countries = []
    sections = []
    subsections = []
    pharma_form_list = []
    pharma_form_type_list = []
    category_list = []
    observation_list = []

    # if medicine status is not active delete from solr index
    if not med.active:
        try:
            solr = SolrConnection(settings.SOLR_URL)
            solr.delete(id=str(med.id))
            response = solr.commit()
        except Exception as ex: 
            return False

        return True

    # index medicine on solr index
    medicine_translations = MedicineLocal.objects.filter(medicine=med.id)
    medicine_list = ['en^%s' % med.name.strip()]
    for translation in medicine_translations:
        medicine_list.append('%s^%s' % (translation.language, translation.name.strip()))
    
    medicine_list = "|".join(medicine_list) # ex.: en^codeine|pt-br^codeína|es^codeína

    # retrieve actives pharmaceutical forms of currente medicine
    pharm_forms = med.pharmaceuticalform_set.filter(active=True)
    for form in pharm_forms:

        # ex. ^enTablet|es^Tableta|pt-br^Comprimido
        pharma_form_type_translations = "|".join( form.pharmaceutical_form_type.get_translations() )
        pharma_form_type_list.append(pharma_form_type_translations)

        # ex. ^enTablet|es^Tableta|pt-br^Comprimido|comp^15 mg/ml
        pharma_form_list.append('%s|comp^%s' % (pharma_form_type_translations, form.composition))

        # create category_list (section and subsection where current pharmaceutical form is used on lists)
        section_pharm_form_list = SectionPharmForm.objects.filter(pharmaceutical_form=form)

        for section_pharm_form in section_pharm_form_list:
            #add observations of current section_pharm_form
            if section_pharm_form.only_for_children:
                observation_list.append('only_for_children')
            if section_pharm_form.specialist_care_for_children:
                observation_list.append('specialist_care_for_children')
            if section_pharm_form.restriction_age:
                observation_list.append('restriction_age')
            if section_pharm_form.best_evidence:
                observation_list.append('best_evidence')
            if section_pharm_form.observation:
                observation_list.append('observation')

            section = Section.objects.get(pk=section_pharm_form.section.id)
            section_translations = "|".join(section.get_translations())

            section_tree = section.get_ancestors()
            
            if section_tree:
                for sec in section_tree:                    
                    category_translations = "|".join(sec.get_translations())
                    if category_translations not in category_list:
                        category_list.append(category_translations)
            
            if section_translations not in category_list:
                category_list.append(section_translations)
   
            list_associated = "|".join( section.list.get_translations() )
            if section.list.type == 'c':
                if list_associated not in countries:                                
                    countries.append(list_associated)
            else:                
                if list_associated not in lists:
                    lists.append(list_associated)

    #check if current medicine have Evidence summaries
    has_evidence = None
    evidence_total = MedicineEvidenceSummary.objects.filter(medicine=med.id).count()
    if evidence_total > 0:
        has_evidence = "true"

    # try to create a connection to a solr server and send medicine
    try:
        solr = SolrConnection(settings.SOLR_URL)
        solr.add(
            id = str(med.id), 
            type = "medicine",
            name = medicine_list,
            pharmaceutical_form = pharma_form_list,        
            pharmaceutical_form_type = pharma_form_type_list,
            list=lists,
            country=countries,
            category=category_list,
            observation=observation_list,
            has_evidence=has_evidence,
        )
        response = solr.commit()
#.........这里部分代码省略.........

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import add [as 别名]

#.........这里部分代码省略.........
            self._sanity_check_batch(batch)

            # stash it away for processing later on
            self.current_batch = batch

            # parse the batch.xml and load up each issue mets file
            doc = etree.parse(batch.validated_batch_url)

            for e in doc.xpath('ndnp:reel', namespaces=ns):

                reel_number = e.attrib['reelNumber'].strip()

                try:
                    reel = models.Reel.objects.get(number=reel_number,
                                                   batch=batch)
                except models.Reel.DoesNotExist as e:
                    reel = models.Reel(number=reel_number, batch=batch)
                    reel.save()

            for e in doc.xpath('ndnp:issue', namespaces=ns):
                mets_url = urlparse.urljoin(batch.storage_url, e.text)

                try:
                    issue, pages = self._load_issue(mets_url)
                except ValueError as e:
                    LOGGER.exception(e)
                    continue

                # commit new changes to the solr index, if we are indexing
                if self.PROCESS_OCR:
                    LOGGER.info("Adding pages to solr index from issue %s", issue.title)
                    for page in pages:
                        LOGGER.debug("indexing ocr for: %s", page.url)
                        self.solr.add(**page.solr_doc)
                        page.indexed = True
                        page.save()

            if self.PROCESS_OCR:
                LOGGER.info("Committing solr index")
                self.solr.commit()

            batch.save()
            msg = "processed %s pages" % batch.page_count
            LOGGER.info(msg)
            event = LoadBatchEvent(batch_name=batch_name, message=msg)
            event.save()
        except Exception as e:
            msg = "unable to load batch: %s" % e
            LOGGER.exception(msg)
            event = LoadBatchEvent(batch_name=batch_name, message=msg)
            event.save()
            try:
                self.purge_batch(batch_name)
            except Exception as pbe:
                LOGGER.error("purge batch failed for failed load batch: %s", pbe)
                LOGGER.exception(pbe)
            raise BatchLoaderException(msg)

        if settings.IS_PRODUCTION:
            batch.released = datetime.now()
            batch.save()

        return batch

    def _get_batch(self, batch_name, batch_source=None, create=False):
        if create:

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import add [as 别名]
             main_graph.parse(f, base="temp_main_manifest.rdf")
             f.close()
     except IOError, e:
         logger.info( str(e))
         logger.info("IOERROR")
         pass
     
     main_solr_doc = gather_document(silo_name, uuid, itemid, main_graph)
                   
     # Add the two dictionaries together              
     main_solr_doc.update(df_solr_doc)
     
     logger.info('Solr_document = ')
     logger.info(main_solr_doc)
     try:
         solr.add(_commit=False, **main_solr_doc)
     except Exception, e :
         logger.error("Error adding document to solr id:%s in silo:%s\n" % (itemid, silo_name))
         try:
            logger.error("%s\n\n" %str(e))
         except:
            pass
         rq.task_failed()
         continue
 # else:
 #     silo_metadata = g.describe_silo(silo_name)
 #     solr_doc = {'id':silo_name, 'silo':silo_name, 'type':'Silo', 'uuid':uuid4().hex}
 #     solr_doc['title'] = ''
 #     if 'title' in silo_metadata:
 #         solr_doc['title'] = silo_metadata['title']
 #     solr_doc['description'] = ''

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import add [as 别名]
#                        'modified': ['2013-06-21 08:56:50.308522'], 
#                        'created': ['2013-06-21 08:56:50.049645'], 
#                        'currentVersion': ['2'], 
#                        'issued': [''], 
#                        'silo': ['DataFinder'], 
#                        'type': ['', 'http://vocab.ox.ac.uk/dataset/schema#DataSet']
#                         }
            # Get the df dictionary  items to be added into solr
            df_solr_doc = gather_document(silo , uuid , item_id,  df_graph )
            
            (resp,respdata) = datastore.doHTTP_GET(resource="/" + silo +"/datasets/" + item_id +"/manifest.rdf")
            text_file = open("sample_redis_main_manifest.rdf", "w")
            text_file.write(respdata)
            text_file.close()
            main_graph=rdflib.Graph()
            try:
                with open("sample_redis_main_manifest.rdf", 'r') as f:
                     main_graph.parse(f, base="sample_redis_manifest.rdf")
            except IOError, e:
                pass
            
            # Get the main dictionary  items to be added into solr
            main_solr_doc = gather_document(silo , uuid , item_id, main_graph )
                          
            # Add the two dictionaries together              
            main_solr_doc.update(df_solr_doc)

            print  main_solr_doc
            solr_doc =  solr.add(_commit=True, **(main_solr_doc))
            solr.commit()