Python Solr.delete_by_id方法代码示例

# 需要导入模块: from utils.search.solr import Solr [as 别名]
# 或者: from utils.search.solr.Solr import delete_by_id [as 别名]
    def handle(self, *args, **options):

        LIMIT = None
        SLICE_SIZE = 500
        solr_sound_ids = []
        solr = Solr(url=settings.SOLR_URL)
        query = SolrQuery()
        query.set_dismax_query("") # Query to get ALL sounds

        print "Retrieving ids from %i to %i"%(0,SLICE_SIZE)
        query.set_query_options(field_list=["id"], rows = SLICE_SIZE, start = 0)
        results = SolrResponseInterpreter(solr.select(unicode(query)))
        solr_sound_ids += list_of_dicts_to_list_of_ids(results.docs)
        total_num_documents = results.num_found

        # Start iterating over other pages (slices)
        if LIMIT:
            number_of_documents = min(LIMIT,total_num_documents)
        else:
            number_of_documents = total_num_documents

        for i in range(SLICE_SIZE,number_of_documents,SLICE_SIZE):
            print "Retrieving ids from %i to %i"%(i,i+SLICE_SIZE)
            query.set_query_options(field_list=["id"], rows = SLICE_SIZE, start = i)
            results = SolrResponseInterpreter(solr.select(unicode(query)))
            solr_sound_ids += list_of_dicts_to_list_of_ids(results.docs)

        solr_sound_ids = sorted(list(set(solr_sound_ids)))
        if LIMIT:
            solr_sound_ids = solr_sound_ids[0:LIMIT]
        print "%i document ids retrieved"%len(solr_sound_ids)
        n_deleted = 0
        print ""
        for count,id in enumerate(solr_sound_ids):
            sys.stdout.write("\rChecking doc %i of %i"%(count,len(solr_sound_ids)))
            sys.stdout.flush()

            if Sound.objects.filter(id=id,moderation_state="OK",processing_state="OK").exists():
                pass
            else:
                # Sound does not exist in the Db or is not properly moderated and processed
                print "\n\t - Deleting sound with id %i from solr index"%id
                solr.delete_by_id(id)
                n_deleted += 1

        print "\n\nDONE! %i sounds deleted from solr index (it may take some minutes to actually see the changes in the page)"%n_deleted

# 需要导入模块: from utils.search.solr import Solr [as 别名]
# 或者: from utils.search.solr.Solr import delete_by_id [as 别名]
    def handle(self,  *args, **options):

        # init
        solr = Solr(settings.SOLR_URL)

        # Get all solr ids
        print "Getting solr ids...",
        solr_ids = get_all_sound_ids_from_solr()
        print "done!"

        # Get ell gaia ids
        print "Getting gaia ids...",
        gaia_ids = Similarity.get_all_sound_ids()
        print "done!"

        print "Getting freesound db data..."
        # Get all moderated and processed sound ids
        queryset = Sound.objects.filter(processing_state='OK', moderation_state='OK').order_by('id').only("id")
        fs_mp = [sound.id for sound in queryset]
        # Get ell moderated, processed and analysed sounds
        queryset = Sound.objects.filter(processing_state='OK', moderation_state='OK', analysis_state='OK').order_by('id').only("id")
        fs_mpa = [sound.id for sound in queryset]
        print "done!"

        print "\nNumber of sounds per index:\n--------------------------"
        print "Solr index\t\t%i" % len(solr_ids)
        print "Gaia index\t\t%i" % len(gaia_ids)
        print "Freesound\t\t%i  (moderated and processed)" % len(fs_mp)
        print "Freesound\t\t%i  (moderated, processed and analyzed)" % len(fs_mpa)

        print "\n\n***************\nSOLR INDEX\n***************\n"
        in_solr_not_in_fs = list(set(solr_ids).intersection(set(set(solr_ids).difference(fs_mp))))
        in_fs_not_in_solr = list(set(fs_mp).intersection(set(set(fs_mp).difference(solr_ids))))
        print "Sounds in solr but not in fs:\t%i" % len(in_solr_not_in_fs)
        print "Sounds in fs but not in solr:\t%i" % len(in_fs_not_in_solr)

        if not options['no-changes']:
            # Mark fs sounds to go processing
            if in_fs_not_in_solr:
                print "Changing is_index_dirty_state of sounds that require it"
                N = len(in_fs_not_in_solr)
                for count, sid in enumerate(in_fs_not_in_solr):
                    sys.stdout.write('\r\tChanging state of sound sound %i of %i         ' % (count+1, N))
                    sys.stdout.flush()
                    sound = Sound.objects.get(id=sid)
                    sound.set_single_field('is_index_dirty', True)

            # Delete sounds from solr that are not in the db
            if in_solr_not_in_fs:
                print "\nDeleting sounds that should not be in solr"
                N = len(in_solr_not_in_fs)
                for count, sid in enumerate(in_solr_not_in_fs):
                    sys.stdout.write('\r\tDeleting sound %i of %i         ' % (count+1, N))
                    sys.stdout.flush()
                    solr.delete_by_id(sid)

        print "\n***************\nGAIA INDEX\n***************\n"
        in_gaia_not_in_fs = list(set(gaia_ids).intersection(set(set(gaia_ids).difference(fs_mpa))))
        in_fs_not_in_gaia = list(set(fs_mpa).intersection(set(set(fs_mpa).difference(gaia_ids))))
        print "Sounds in gaia but not in fs:\t%i" % len(in_gaia_not_in_fs)
        print "Sounds in fs but not in gaia:\t%i  (only considering sounds correctly analyzed)" % len(in_fs_not_in_gaia)
        #Similarity.save()

        if not options['no-changes']:
            # Mark fs sounds to go processing
            if in_fs_not_in_gaia:
                print "Changing similarity_state of sounds that require it"
                N = len(in_fs_not_in_gaia)
                for count, sid in enumerate(in_fs_not_in_gaia):
                    sys.stdout.write('\r\tChanging state of sound %i of %i         ' % (count+1, N))
                    sys.stdout.flush()
                    sound = Sound.objects.get(id=sid)
                    sound.set_similarity_state('PE')

            # Delete sounds from gaia that are not in the db
            if in_gaia_not_in_fs:
                print "\nDeleting sounds that should not be in gaia"
                N = len(in_gaia_not_in_fs)
                for count, sid in enumerate(in_gaia_not_in_fs):
                    sys.stdout.write('\r\tDeleting sound %i of %i         ' % (count+1, N))
                    sys.stdout.flush()
                    Similarity.delete(sid)