Python SolrConnection.commit方法代码示例

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import commit [as 别名]
 def handle(self, **options):
     solr = SolrConnection(SOLR_URL)
     if options["user"]:
         solr.delete_query("user:%s" % options["user"])
     else:
         solr.delete_query("id:[* TO *]")
     solr.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import commit [as 别名]
def index_titles(since=None):
    """index all the titles and holdings that are modeled in the database
    if you pass in a datetime object as the since parameter only title
    records that have been created since that time will be indexed.
    """
    cursor = connection.cursor()
    solr = SolrConnection(settings.SOLR)
    if since:
        cursor.execute("SELECT lccn FROM core_title WHERE created >= '%s'" % since)
    else:
        solr.delete_query('type:title')
        cursor.execute("SELECT lccn FROM core_title")

    count = 0
    while True:
        row = cursor.fetchone()
        if row is None:
            break
        title = models.Title.objects.get(lccn=row[0])
        index_title(title, solr)
        count += 1
        if count % 100 == 0:
            LOGGER.info("indexed %s titles", count)
            reset_queries()
            solr.commit()
    solr.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import commit [as 别名]
 def solr_delete(self):
     """
     Remove from solr index
     """
     solr_conn = SolrConnection(settings.SOLR_URL, persistent=False)
     solr_conn.delete_query('id:%s' % self.id)
     solr_conn.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import commit [as 别名]
 def solr_reindex (self):
     """
     Reindex all entries.  Used when switching to/from "private" status.
     """
     solr_conn = SolrConnection(settings.SOLR_URL)
     # Start by deleting 'em all
     solr_conn.delete_query('user:%s' % self.user.id)
     entries = Entry.objects.filter(user=self.user)
     docs = []
     # Arbitrary assignment of a constant, here.
     SLICE_SIZE = 50
     slices = [x for x in range(entries.count()) \
         if x % SLICE_SIZE == 0]
     for s in slices:
         entry_slice = entries[s:s+SLICE_SIZE]
         for entry in entry_slice:
             docs.append(entry.solr_doc)
             if len(docs) == SLICE_SIZE:
                 try:
                     solr_conn.add_many(docs)
                 except:
                     # should log appropriately, huh
                     pass
                 del(docs)
                 docs = []
     # Don't miss the leftovers
     solr_conn.add_many(docs)
     solr_conn.commit()
     solr_conn.optimize()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import commit [as 别名]
 def solr_index(self):
     """
     Write out to solr
     """
     solr_conn = SolrConnection(settings.SOLR_URL, persistent=False)
     solr_conn.add(**self.solr_doc)
     solr_conn.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import commit [as 别名]
 def handle(self, **options):
     solr = SolrConnection(settings.SOLR)
     if options['batch']:
         solr.delete_query('batch: %s' % options['batch'])
     else:
         solr.delete_query('id:[* TO *]')
     solr.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import commit [as 别名]
class Command(BaseCommand):
    user_option = optparse.make_option('--user',
        action='store', dest='user',
        help='name of user whose entries to purge')
    option_list = BaseCommand.option_list + (user_option,)
    help = "index all or user-specific entries in solr"
    args = 'an optional username'

    def handle(self, *args, **options):
        self.solr = SolrConnection(SOLR_URL)
        self.cursor = connection.cursor()
        if options['user']:
            print "indexing user"
            self.index_entries(user=options['user'])
        else:
            print 'indexing everything'
            self.index_entries()
        print 'committing'
        self.solr.commit()
        print 'optimizing'
        self.solr.optimize()

    def index_entries(self, user=''):
        counter = 0
        entries = m.Entry.objects.all()
        if user:
            entries = entries.filter(user__username=user)
        docs = []
        print 'entry count:', entries.count()
        SLICE_SIZE = MAX_DOCS_PER_ADD * COMMIT_FREQUENCY 
        slices = [x for x in range(entries.count()) \
            if x % SLICE_SIZE == 0]
        for s in slices:
            print 'indexing %s to %s...' % (s, s+SLICE_SIZE)
            entry_slice = entries[s:s+SLICE_SIZE]
            for entry in entry_slice:
                counter += 1
                docs.append(entry.solr_doc)
                if len(docs) == MAX_DOCS_PER_ADD:
                    try:
                        self.solr.add_many(docs)
                    except:
                        print 'BAD RECORD:', [d['id'] for d in docs]
                    del(docs)
                    docs = []
                    reset_queries()
                    if counter % (COMMIT_FREQUENCY * MAX_DOCS_PER_ADD) == 0:
                        print 'committing at count:', counter
                        self.solr.commit()
        # Don't miss the leftovers
        self.solr.add_many(docs)

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import commit [as 别名]
def _refresh(field=None, data=None, path = None, isCron = None):
	from solr import SolrConnection
	from ID3 import *
	s = SolrConnection(SOLR_URL)
	if path and path != '*':
		#called by user		
		pathsArr = path.split(',')		
	else:
		#called from cron		
		pathsArr = folderpaths
	matches = []
	#handles modify, add
	#deletion will be handled in search when file in solr but not in path
	time.time()
	for path in pathsArr:
		for root, dirnames, filenames in os.walk(path):
			for extension in ['txt', 'log', 'py', 'pl', 'sql', 'mp3']:
				for filename in fnmatch.filter(filenames, '*.' + extension):				
					fullName = os.path.join(root, filename)
					if os.path.getsize(fullName) > 8800000:
						continue
					#print fullName
					if not isCron or (time.time() - os.path.getmtime(fullName) < 24*60*60):				
						try:
							#data = open(fullName, 'r').read().decode('raw_unicode_escape').replace('\n',' ').replace('\t',' ')
							if filename.endswith(('.txt', '.log', '.py', '.pl', '.sql')):								
								data = open(fullName, 'r').read()
								data = filterTxt(data)
							else:								
								audiofile = ID3(fullName)
								audiofilekeys = audiofile.keys()
								if 'TITLE' in audiofilekeys:
									data = audiofile['TITLE'] + " "
								if 'ARTIST' in audiofilekeys:
									data += audiofile['ARTIST'] + " "
								if 'ALBUM' in audiofilekeys:
									data += audiofile['ALBUM'] + " "
								if not data:
									data = ''
								data = data.strip()
							fullName = filterTxt(fullName)
							filename = filterTxt(filename)						
							s.add(id = fullName, name = filename, txt = data)
							s.commit()
						except:																	
							pass
							#print data
							#print traceback.format_exc()
							#print fullName	
							#sys.exit()					
						gc.collect()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import commit [as 别名]
	def search(self, **kwargs):

		query = kwargs['q']
		api_key = "aac5b38a36513510000ef3286494fc6d"

		url = urllib2.urlopen("http://tinysong.com/s/%s/?format=json&key=%s" % (urllib2.quote(query), api_key))
		response = json.loads(url.read())

		# TODO: Remove redundancy between results and tracks?
		results = []
		tracks = []
		for song in response:

			source_id = 'grooveshark'

			result = {
				'artist': song['ArtistName'],
				'album': song['AlbumName'],
				'title': song['SongName'],
				'sources': [
					{
						'sourceid': source_id,
						'trackid': '%s' % song['SongID']
					}
				]
			}
			results.append(result)

			track = {
				'id': 'track_%s_%s' % (source_id, song['SongID']),
				'type': 'track',

				'track_title': song['SongName'],
				'track_artist': song['ArtistName'],
				'track_album': song['AlbumName'],

				'request_source_id': source_id,
				'request_track_id': song['SongID'],
			}
			tracks.append(track)

		# Register the songs in the search engine
		solr = SolrConnection(settings.SOLR_URL)
		solr.add_many(tracks)
		solr.commit()
		solr.close()

		cherrypy.response.headers['Content-Type'] = 'application/json; charset=utf-8'
		return json.dumps(results, ensure_ascii=False, indent=4).encode('utf-8')

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import commit [as 别名]
def index_missing_pages():
    """
    index all pages that are missing from solr in the database
    """
    solr = SolrConnection(settings.SOLR)
    count = 0
    pages = models.Page.objects.filter(indexed=False).all()
    number_of_pages = len(pages)
    for page in pages:
        LOGGER.info("[%s of %s] indexing page: %s", count, number_of_pages, page.url)
        solr.add(**page.solr_doc)
        count += 1
        page.indexed = True
        page.save()
    solr.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import commit [as 别名]
def index_evidence(evidence):
    evidence_medicine_list = []

    evidence_medicine = MedicineEvidenceSummary.objects.filter(evidence=evidence.id)
    for evimed in evidence_medicine: 
        if evimed.medicine.name not in evidence_medicine_list:
                evidence_medicine_list.append(evimed.medicine.name)

    # try to create a connection to a solr server and send medicine
    try:
        solr = SolrConnection(settings.SOLR_URL)
        solr.add(
            id = "evidence-%s-%s" % (evidence.language, evidence.id), 
            type = "evidence",
            title = evidence.title,            
            description = evidence.description,
            context = evidence.context,
            question = evidence.question,
            link = evidence.link,
            file = evidence.file,
            language = evidence.language,
            evidence_medicine = evidence_medicine_list,
        )
        response = solr.commit()
    except Exception as ex: 
        return False

    return True

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import commit [as 别名]
	def finished(self, **kwargs):

		source_id = kwargs['source_id']
		track_id = kwargs['track_id']

		# Build up a Solr query
		filters = []
		filters.append('type:request')
		filters.append('channel_id:%s' % self.channel_id)
		filters.append('request_source_id:%s' % source_id)
		filters.append('request_track_id:%s' % track_id)

		# Make the request to Solr
		solr = SolrConnection(settings.SOLR_URL)
		solr.delete_query(' AND '.join(filters))
		solr.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import commit [as 别名]
    def create(self, **kwargs):

        # Collect the channel details
        name = kwargs["name"]
        pos = kwargs["pos"]

        # Create the channel in the search engine
        doc = {"id": "channel_%s" % (name,), "type": "channel", "channel_id": name, "channel_location": pos}

        solr = SolrConnection(settings.SOLR_URL)
        solr.add_many([doc])
        solr.commit()
        solr.close()

        # Create the channel in the URL hierarchy
        self.__dict__[name] = ChannelResource.Channel(name)

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import commit [as 别名]
def index_pages():
    """index all the pages that are modeled in the database
    """
    solr = SolrConnection(settings.SOLR)
    solr.delete_query('type:page')
    cursor = connection.cursor()
    cursor.execute("SELECT id FROM core_page")
    count = 0
    while True:
        row = cursor.fetchone()
        if row is None:
            break
        page = models.Page.objects.get(id=row[0])
        LOGGER.info("[%s] indexing page: %s", count, page.url)
        solr.add(**page.solr_doc)
        count += 1
        if count % 100 == 0:
            reset_queries()
    solr.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import commit [as 别名]
    def setUpClass(cls):
        # First, add a folio to Solr so that the image_uri can be retrieved during the MEI conversion
        # Using curl here because it turned out to be easier than solrconn.add and gives better error messages
        os.system("curl {0}/update/?commit=true -H 'Content-Type: text/xml' -d '<add><doc>\
        <field name=\"id\">testid</field>\
        <field name=\"type\">cantusdata_folio</field>\
        <field name=\"manuscript_id\">{1}</field>\
        <field name=\"number\">{2}</field>\
        <field name=\"image_uri\">{3}</field>\
        </doc></add>'".format(settings.SOLR_SERVER, MEI_FIXTURE_ID, MEI_FIXTURE_FOLIO, MEI_FIXTURE_URI))

        docs = list(MEIConverter.process_file(MEI_FIXTURE, MEI_FIXTURE_SIGLUM, MEI_FIXTURE_ID))

        # Sanity check
        solrconn = SolrConnection(settings.SOLR_SERVER)
        prequery = solrconn.query('type:cantusdata_music_notation AND manuscript:' + MEI_FIXTURE_SIGLUM)
        assert prequery.numFound == 0, 'MEI was already in the database when loading the test fixture'

        solrconn.add_many(docs)
        solrconn.commit()