Python SolrConnection.delete_query方法代码示例

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import delete_query [as 别名]
 def handle(self, **options):
     solr = SolrConnection(settings.SOLR)
     if options['batch']:
         solr.delete_query('batch: %s' % options['batch'])
     else:
         solr.delete_query('id:[* TO *]')
     solr.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import delete_query [as 别名]
def index_titles(since=None):
    """index all the titles and holdings that are modeled in the database
    if you pass in a datetime object as the since parameter only title
    records that have been created since that time will be indexed.
    """
    cursor = connection.cursor()
    solr = SolrConnection(settings.SOLR)
    if since:
        cursor.execute("SELECT lccn FROM core_title WHERE created >= '%s'" % since)
    else:
        solr.delete_query('type:title')
        cursor.execute("SELECT lccn FROM core_title")

    count = 0
    while True:
        row = cursor.fetchone()
        if row is None:
            break
        title = models.Title.objects.get(lccn=row[0])
        index_title(title, solr)
        count += 1
        if count % 100 == 0:
            LOGGER.info("indexed %s titles", count)
            reset_queries()
            solr.commit()
    solr.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import delete_query [as 别名]
 def handle(self, **options):
     solr = SolrConnection(SOLR_URL)
     if options["user"]:
         solr.delete_query("user:%s" % options["user"])
     else:
         solr.delete_query("id:[* TO *]")
     solr.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import delete_query [as 别名]
 def solr_reindex (self):
     """
     Reindex all entries.  Used when switching to/from "private" status.
     """
     solr_conn = SolrConnection(settings.SOLR_URL)
     # Start by deleting 'em all
     solr_conn.delete_query('user:%s' % self.user.id)
     entries = Entry.objects.filter(user=self.user)
     docs = []
     # Arbitrary assignment of a constant, here.
     SLICE_SIZE = 50
     slices = [x for x in range(entries.count()) \
         if x % SLICE_SIZE == 0]
     for s in slices:
         entry_slice = entries[s:s+SLICE_SIZE]
         for entry in entry_slice:
             docs.append(entry.solr_doc)
             if len(docs) == SLICE_SIZE:
                 try:
                     solr_conn.add_many(docs)
                 except:
                     # should log appropriately, huh
                     pass
                 del(docs)
                 docs = []
     # Don't miss the leftovers
     solr_conn.add_many(docs)
     solr_conn.commit()
     solr_conn.optimize()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import delete_query [as 别名]
 def solr_delete(self):
     """
     Remove from solr index
     """
     solr_conn = SolrConnection(settings.SOLR_URL, persistent=False)
     solr_conn.delete_query('id:%s' % self.id)
     solr_conn.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import delete_query [as 别名]
	def finished(self, **kwargs):

		source_id = kwargs['source_id']
		track_id = kwargs['track_id']

		# Build up a Solr query
		filters = []
		filters.append('type:request')
		filters.append('channel_id:%s' % self.channel_id)
		filters.append('request_source_id:%s' % source_id)
		filters.append('request_track_id:%s' % track_id)

		# Make the request to Solr
		solr = SolrConnection(settings.SOLR_URL)
		solr.delete_query(' AND '.join(filters))
		solr.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import delete_query [as 别名]
def index_pages():
    """index all the pages that are modeled in the database
    """
    solr = SolrConnection(settings.SOLR)
    solr.delete_query('type:page')
    cursor = connection.cursor()
    cursor.execute("SELECT id FROM core_page")
    count = 0
    while True:
        row = cursor.fetchone()
        if row is None:
            break
        page = models.Page.objects.get(id=row[0])
        LOGGER.info("[%s] indexing page: %s", count, page.url)
        solr.add(**page.solr_doc)
        count += 1
        if count % 100 == 0:
            reset_queries()
    solr.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import delete_query [as 别名]
def delete_title(title):
    solr = SolrConnection(settings.SOLR)
    q = '+type:title +id:%s' % title.solr_doc['id']
    solr.delete_query(q)
    LOGGER.info("deleted title %s from the index", title)

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import delete_query [as 别名]
 def tearDownClass(cls):
     solrconn = SolrConnection(settings.SOLR_SERVER)
     solrconn.delete_query('type:cantusdata_music_notation AND manuscript:' + MEI_FIXTURE_SIGLUM)
     solrconn.delete_query('type:cantusdata_folio AND manuscript_id:{0}'.format(MEI_FIXTURE_ID))
     solrconn.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import delete_query [as 别名]
                silo_metadata = {}
                tries = 0
                while tries < 5:
                    response = db.getSiloState(silo_name)
                    if db.good(response):
                        silo_metadata = response.results
                        break
                    else:
                        tries += 1
                solr_doc["title"] = ""
                if "title" in silo_metadata:
                    solr_doc["title"] = silo_metadata["title"]
                solr_doc["description"] = ""
                if "description" in silo_metadata:
                    solr_doc["description"] = silo_metadata["description"]
                solr.add(_commit=False, **solr_doc)
            rq.task_complete()
        elif msg["type"] == "d":
            # Deletion
            itemid = msg.get("id", None)
            if itemid:
                logger.info("Got deletion message on id:%s in silo:%s" % (itemid, silo_name))
                query = 'silo:"%s" AND id:"%s"' % (silo_name, itemid)
                solr.delete_query(query)
            elif silo_name:
                logger.info("Got deletion message on silo:%s" % silo_name)
                query = 'silo:"%s"' % silo_name
                solr.delete_query(query)
                # solr.commit()
            rq.task_complete()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import delete_query [as 别名]

#.........这里部分代码省略.........
            except models.Language.DoesNotExist:
                LOGGER.warn("Language %s does not exist in the database. Defaulting to English.", lang)
                # default to english as per requirement
                language = models.Language.objects.get(code='eng')
            ocr.language_texts.create(language=language)
            lang_text_solr[language.code] = text

        page.ocr = ocr
        page.lang_text = lang_text_solr
        page.save()
        return page

    def _process_coordinates(self, page, coords):
        LOGGER.debug("writing out word coords for %s", page.url)

        fd, path = tempfile.mkstemp(text="w", suffix=".coordinates", dir=settings.TEMP_STORAGE)  # get a temp file in case the coordinates dir is a NFS or S3 mount which have poor multiple write performance
        f = open(path, "w")
        f.write(gzip_compress(json.dumps(coords)))
        f.close()
        os.close(fd)
        final_path = models.coordinates_path(page._url_parts())
        try:
            shutil.move(path, final_path)
        except Exception:
            LOGGER.warn("Could not move coordinates to [%s]. Waiting 5 seconds and trying again in case of network mount", final_path)
            time.sleep(5)
            shutil.move(path, final_path)

    def process_coordinates(self, batch_path):
        LOGGER.info("process word coordinates for batch at %s", batch_path)
        dirname, batch_name = os.path.split(batch_path.rstrip("/"))
        if dirname:
            batch_source = None
        else:
            batch_source = urlparse.urljoin(settings.BATCH_STORAGE, batch_name)
            if not batch_source.endswith("/"):
                batch_source += "/"
        batch_name = _normalize_batch_name(batch_name)
        try:
            batch = self._get_batch(batch_name, batch_source, create=False)
            self.current_batch = batch
            for issue in batch.issues.all():
                for page in issue.pages.all():
                    if not page.ocr_filename:
                        LOGGER.warn("Batch [%s] has page [%s] that has no OCR. Skipping processing coordinates for page." % (batch_name, page))
                    else:
                        url = urlparse.urljoin(self.current_batch.storage_url,
                                               page.ocr_filename)
                        LOGGER.debug("Extracting OCR from url %s", url)
                        lang_text, coords = ocr_extractor(url)
                        self._process_coordinates(page, coords)
        except Exception as e:
            msg = "unable to process coordinates for batch: %s" % e
            LOGGER.exception(msg)
            raise BatchLoaderException(msg)

    def storage_relative_path(self, path):
        """returns a relative path for a given file path within a batch, so
        that storage can be re-homed without having to rewrite paths in the db
        """
        rel_path = path.replace(self.current_batch.storage_url, '')
        return rel_path

    @transaction.atomic
    def purge_batch(self, batch_name):
        event = LoadBatchEvent(batch_name=batch_name, message="starting purge")
        event.save()

        try:
            batch = self._get_batch(batch_name)
            self._purge_batch(batch)
            event = LoadBatchEvent(batch_name=batch_name, message="purged")
            event.save()
            # clean up symlinks if exists
            link_name = os.path.join(settings.BATCH_STORAGE, batch_name)
            if os.path.islink(link_name):
                LOGGER.info("Removing symlink %s", link_name)
                os.remove(link_name)
        except Exception as e:
            msg = "purge failed: %s" % e
            LOGGER.exception(msg)
            event = LoadBatchEvent(batch_name=batch_name, message=msg)
            event.save()
            raise BatchLoaderException(msg)

    def _purge_batch(self, batch):
        batch_name = batch.name
        # just delete batch causes memory to bloat out
        # so we do it piece-meal
        for issue in batch.issues.all():
            for page in issue.pages.all():
                page.delete()
                # remove coordinates
                if os.path.exists(models.coordinates_path(page._url_parts())):
                    os.remove(models.coordinates_path(page._url_parts()))
            issue.delete()
        batch.delete()
        if self.PROCESS_OCR:
            self.solr.delete_query('batch:"%s"' % batch_name)
            self.solr.commit()

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import delete_query [as 别名]
def search(field, data, path, hlength, mode):
	from termcolor import colored
	from solr import SolrConnection
	#hlength = int(hlength)
	#search solr, get filePath, do a grep and show the line
	#print 'search'
	s = SolrConnection(SOLR_URL)
	if field == 'name':
		query = 'name:"' + data + '"'
		response = s.query(query)
	elif field == 'txt':
		query = 'txt:"' + data + '"'		
		#response = s.query(query, hl=True, hl.q='txt:bandits', hl.fl='txt', hl.fragsize=50, hl.preserveMulti=True, hl.snippets=100)
		if hlength:
			response = s.query(query, fl='id,name', highlight=True, fields = 'txt', hl_q=query, hl_fragsize=hlength, hl_snippets=1000, hl_bs_type = 'SENTENCE')																																	 
		else:
			response = s.query(query, fl='id,name')
	else:
		query = 'name:"' + data + '" OR txt:"' + data + '"'
		#response = s.query(query, hl=True, hl.q='txt:bandits', hl.fl='txt', hl.fragsize=50, hl.preserveMulti=True, hl.snippets=100)
		if hlength:
			response = s.query(query, fl='id,name', highlight=True, fields = 'txt', hl_q=query, hl_fragsize=hlength, hl_snippets=1000, hl_bs_type = 'SENTENCE')
		else:
			response = s.query(query, fl='id,name')

	#print query
	#print response.__dict__
	#print response.highlighting
	if hlength and field != 'name':
		hlength = int(hlength)			
		for id in response.highlighting:
			if os.path.isfile(id):			
				if response.highlighting[id]:
					for txt in response.highlighting[id]['txt']:
						txt = txt.strip()
						startpos = txt.index('<em>')
						endpos = txt.rindex('</em>')
						print (txt[:startpos] + colored(txt[startpos+4:endpos], 'red') + txt[endpos+5:]).replace('<em>', '').replace('</em>', '')
				else:
					fdata = open(id, 'r').read().decode('raw_unicode_escape').replace('\n',' ').replace('\t',' ')
					fdata = filter(lambda x: x in string.printable, fdata)
					for m in re.finditer( data, fdata ):
						start = m.start()-hlength
						if start < 0 :
							start = 0					
						end = m.end() + hlength
						if end > len(fdata):
							end = len(fdata)

						print (fdata[start:m.start()] + colored(fdata[m.start():m.end()], 'red') + fdata[m.end():end]).replace('<em>', '').replace('</em>', '')
				if id.endswith(('.mp3')):
					if mode == 'slow':
						x = raw_input('press `y` to play, `n` to move forward \n')
						if x == 'y':
							subprocess.call(["afplay", id])
				else:					
					print '\t To open the file press cmd + double click '
					print colored("file://"+id, 'blue')			
					print '\n \n'
					if mode == 'slow':
						raw_input('press any key to continue \n')

			else:
				s.delete_query('id:'+id)
	else:
		for hit in response.results:			
			if hit['id']:
				if hit['id'].endswith(('.mp3')):
					if mode == 'slow':
						x = raw_input('press `y` to play, `n` to move forward \n')
						if x == 'y':
							subprocess.call(["afplay", hit['id']])
				else:					
					print '\t To open the file press cmd + double click '
					print colored("file://"+hit['id'], 'blue')			
					print '\n \n'
					if mode == 'slow':
						raw_input('press any key to continue \n')
			else:
				s.delete_query('id:'+hit['id'])

# 需要导入模块: from solr import SolrConnection [as 别名]
# 或者: from solr.SolrConnection import delete_query [as 别名]
def uninstall(field, data, path, length, mode):	
	from solr import SolrConnection
	s = SolrConnection(SOLR_URL)
	s.delete_query('id:*')
	s.commit()