Python search_engine_factory.SearchEngineFactory类代码示例

def prepare_resource_relations_index(create=False):
    """
    Creates the settings and mappings in Elasticsearch to support related resources

    """

    index_settings = {
        'mappings': {
            'all': {
                'properties': {
                    'resourcexid': {'type': 'keyword'},
                    'notes': {'type': 'text'},
                    'relationshiptype': {'type': 'keyword'},
                    'resourceinstanceidfrom': {'type': 'keyword'},
                    'resourceinstanceidto': {'type': 'keyword'},
                    'created': {'type': 'keyword'},
                    'modified': {'type': 'keyword'}
                }
            }
        }
    }

    if create:
        se = SearchEngineFactory().create()
        se.create_index(index='resource_relations', body=index_settings, ignore=400)

    return index_settings

 def get_scheme_id(self):
     se = SearchEngineFactory().create()
     result = se.search(index='concept_labels', id=self.id)
     if result['found']:
         return Concept(result['_type'])
     else:
         return None

    def prepare_search_index(self, resource_type_id, create=False):
        """
        Creates the settings and mappings in Elasticsearch to support resource search

        """

        index_settings = super(Resource, self).prepare_search_index(resource_type_id, create=False)

        index_settings['mappings'][resource_type_id]['properties']['date_groups'] = { 
            'properties' : {
                'conceptid': {'type' : 'string', 'index' : 'not_analyzed'}
            }
        }
        
        #index_settings['mappings'][resource_type_id]['properties']['measurement_groups'] = { 
        #    'properties' : {
        #        'conceptid': {'type' : 'string', 'index' : 'not_analyzed'}
        #    }
        #}

        if create:
            se = SearchEngineFactory().create()
            try:
                se.create_index(index='entity', body=index_settings)
            except:
                index_settings = index_settings['mappings']
                se.create_mapping(index='entity', doc_type=resource_type_id, body=index_settings)

    def test_bulk_delete(self):
        """
        Test bulk deleting of documents in Elasticsearch

        """

        se = SearchEngineFactory().create()
        # se.create_index(index='test')

        for i in range(10):
            x = {
                'id': i,
                'type': 'prefLabel',
                'value': 'test pref label',
            }
            se.index_data(index='test', doc_type='test', body=x, idfield='id', refresh=True)
            y = {
                'id': i + 100,
                'type': 'altLabel',
                'value': 'test alt label',
            }
            se.index_data(index='test', doc_type='test', body=y, idfield='id', refresh=True)


        query = Query(se, start=0, limit=100)
        match = Match(field='type', query='altLabel')
        query.add_query(match)

        query.delete(index='test', refresh=True)

        self.assertEqual(se.es.count(index='test', doc_type='test')['count'], 10)

def resource_manager(request, resourcetypeid='', form_id='default', resourceid=''):

    if resourceid != '':
        resource = Resource(resourceid)
    elif resourcetypeid != '':
        resource = Resource({'entitytypeid': resourcetypeid})

    if form_id == 'default':
        form_id = resource.form_groups[0]['forms'][0]['id']

    form = resource.get_form(form_id)

    if request.method == 'DELETE':
        resource.delete_index()
        se = SearchEngineFactory().create()
        realtionships = resource.get_related_resources(return_entities=False)
        for realtionship in realtionships:
            se.delete(index='resource_relations', doc_type='all', id=realtionship.resourcexid)
            realtionship.delete()
        resource.delete()
        return JSONResponse({ 'success': True })

    if request.method == 'POST':
        data = JSONDeserializer().deserialize(request.POST.get('formdata', {}))
        form.update(data, request.FILES)

        with transaction.atomic():
            if resourceid != '':
                resource.delete_index()
            resource.save(user=request.user)
            resource.index()
            resourceid = resource.entityid

            return redirect('resource_manager', resourcetypeid=resourcetypeid, form_id=form_id, resourceid=resourceid)

    min_max_dates = models.Dates.objects.aggregate(Min('val'), Max('val'))
    
    if request.method == 'GET':
        if form != None:
            lang = request.GET.get('lang', settings.LANGUAGE_CODE)
            form.load(lang)
            return render(request, 'resource-manager.htm', {
                'form': form,
                'formdata': JSONSerializer().serialize(form.data),
                'form_template': 'views/forms/' + form_id + '.htm',
                'form_id': form_id,
                'resourcetypeid': resourcetypeid,
                'resourceid': resourceid,
                'main_script': 'resource-manager',
                'active_page': 'ResourceManger',
                'resource': resource,
                'resource_name': resource.get_primary_name(),
                'resource_type_name': resource.get_type_name(),
                'form_groups': resource.form_groups,
                'min_date': min_max_dates['val__min'].year if min_max_dates['val__min'] != None else 0,
                'max_date': min_max_dates['val__max'].year if min_max_dates['val__min'] != None else 1,
                'timefilterdata': JSONSerializer().serialize(Concept.get_time_filter_data()),
            })
        else:
            return HttpResponseNotFound('<h1>Arches form not found.</h1>')

 def get_scheme_id(self):
     se = SearchEngineFactory().create()
     result = se.search(index='strings', doc_type='concept', id=self.id)
     if result['found']:
         return Concept(result['top_concept'])
     else:
         return None

    def prepare_resource_relations_index(self, create=False):
        """
        Creates the settings and mappings in Elasticsearch to support related resources

        """

        index_settings = { 
            'mappings':{
                'all': {
                    'properties': {
                        'resourcexid': {'type': 'long'},
                        'notes': { 'type': 'string'},
                        'relationshiptype': {'type': 'string', 'index' : 'not_analyzed'},
                        'entityid2': {'type': 'string', 'index' : 'not_analyzed'},
                        'entityid1': {'type': 'string', 'index' : 'not_analyzed'}
                    }  
                }
            }
        }    

        if create:
            se = SearchEngineFactory().create()
            se.create_index(index='resource_relations', body=index_settings, ignore=400)

        return index_settings

def related_resources(request, resourceid):

    ## get allowed resource types based on permissions
    allowedtypes = get_allowed_types(request)
    is_anon = False
    if request.user.username == "anonymous":
        is_anon = True
    
    if request.method == 'GET':
        lang = request.GET.get('lang', settings.LANGUAGE_CODE)
        start = request.GET.get('start', 0)
        resources = get_related_resources(resourceid, lang, start=start, limit=15, allowedtypes=allowedtypes, is_anon=is_anon)
        return JSONResponse(resources, indent=4)
    
    if 'edit' in request.user.user_groups and request.method == 'DELETE':
        se = SearchEngineFactory().create()
        data = JSONDeserializer().deserialize(request.body) 
        entityid1 = data.get('entityid1')
        entityid2 = data.get('entityid2')
        resourcexid = data.get('resourcexid')
        realtionshiptype = data.get('realtionshiptype')
        resource = Resource(entityid1)
        resource.delete_resource_relationship(entityid2, realtionshiptype)
        se.delete(index='resource_relations', doc_type='all', id=resourcexid)
        return JSONResponse({ 'success': True })

    def get_related_resources(self, lang='en-US', limit=1000, start=0):
        """
        Returns an object that lists the related resources, the relationship types, and a reference to the current resource

        """

        ret = {
            'resource_instance': self,
            'resource_relationships': [],
            'related_resources': []
        }
        se = SearchEngineFactory().create()
        query = Query(se, limit=limit, start=start)
        bool_filter = Bool()
        bool_filter.should(Terms(field='resourceinstanceidfrom', terms=self.resourceinstanceid))
        bool_filter.should(Terms(field='resourceinstanceidto', terms=self.resourceinstanceid))
        query.add_query(bool_filter)
        resource_relations = query.search(index='resource_relations', doc_type='all')
        ret['total'] = resource_relations['hits']['total']
        instanceids = set()
        for relation in resource_relations['hits']['hits']:
            relation['_source']['preflabel'] = get_preflabel_from_valueid(relation['_source']['relationshiptype'], lang)
            ret['resource_relationships'].append(relation['_source'])
            instanceids.add(relation['_source']['resourceinstanceidto'])
            instanceids.add(relation['_source']['resourceinstanceidfrom'])
        if len(instanceids) > 0:
            instanceids.remove(str(self.resourceinstanceid))

        related_resources = se.search(index='resource', doc_type='_all', id=list(instanceids))
        if related_resources:
            for resource in related_resources['docs']:
                ret['related_resources'].append(resource['_source'])

        return ret

def get_related_resources(resourceid, lang, limit=1000, start=0):
    ret = {
        'resource_relationships': [],
        'related_resources': []
    }
    se = SearchEngineFactory().create()

    query = Query(se, limit=limit, start=start)
    query.add_filter(Terms(field='entityid1', terms=resourceid).dsl, operator='or')
    query.add_filter(Terms(field='entityid2', terms=resourceid).dsl, operator='or')
    resource_relations = query.search(index='resource_relations', doc_type='all')
    ret['total'] = resource_relations['hits']['total']

    entityids = set()
    for relation in resource_relations['hits']['hits']:
        relation['_source']['preflabel'] = get_preflabel_from_valueid(relation['_source']['relationshiptype'], lang)
        ret['resource_relationships'].append(relation['_source'])
        entityids.add(relation['_source']['entityid1'])
        entityids.add(relation['_source']['entityid2'])
    if len(entityids) > 0:
        entityids.remove(resourceid)   

    related_resources = se.search(index='entity', doc_type='_all', id=list(entityids))
    if related_resources:
        for resource in related_resources['docs']:
            ret['related_resources'].append(resource['_source'])

    return ret

def get_preflabel_from_valueid(valueid, lang):

    se = SearchEngineFactory().create()
    concept_label = se.search(index='concept_labels', id=valueid)
    if concept_label['found']:
#         print "ConceptID from ValueID: %s" % get_concept_label_from_valueid(valueid)
        return get_preflabel_from_conceptid(get_concept_label_from_valueid(valueid)['conceptid'], lang)

def index_resources():
    """
    Deletes any existing indicies from elasticsearch related to resources 
    and then indexes all resources from the database

    """

    result_summary = {}
    se = SearchEngineFactory().create()

    # clear existing indexes
    for index_type in ['resource_relations', 'entity', 'resource', 'maplayers']:
        se.delete_index(index=index_type)
    se.delete(index='term', body='{"query":{"bool":{"must":[{"constant_score":{"filter":{"missing":{"field":"value.options.conceptid"}}}}],"must_not":[],"should":[]}}}')

    Resource().prepare_term_index(create=True)

    cursor = connection.cursor()
    cursor.execute("""select entitytypeid from data.entity_types where isresource = TRUE""")
    resource_types = cursor.fetchall()
    Resource().prepare_resource_relations_index(create=True)

    for resource_type in resource_types:
        Resource().prepare_search_index(resource_type[0], create=True)
    
    index_resources_by_type(resource_types, result_summary)

    se.es.indices.refresh(index='entity')
    for resource_type in resource_types:
        result_summary[resource_type[0]]['indexed'] = se.es.count(index="entity", doc_type=resource_type[0])['count']

    print '\nResource Index Results:'
    for k, v in result_summary.iteritems():
        status = 'Passed' if v['database'] == v['indexed'] else 'failed'
        print "Status: {0}, Resource Type: {1}, In Database: {2}, Indexed: {3}".format(status, k, v['database'], v['indexed'])

def index_resources_by_type(resource_types, result_summary):
    """
    Collects and indexes all resources

    """

    for resource_type in resource_types:
        resources = archesmodels.Entities.objects.filter(entitytypeid = resource_type)
        print "Indexing {0} {1} resources".format(len(resources), resource_type[0])
        result_summary[resource_type[0]] = {'database':len(resources), 'indexed':0}
        errors = []
        for resource in resources:
            try:
                resource = Resource().get(resource.entityid)
                resource.index()
            except Exception as e:
                if e not in errors:
                    errors.append(e)
        if len(errors) > 0:
            print errors[0], ':', len(errors)

        se = SearchEngineFactory().create()
        related_resource_records = archesmodels.RelatedResource.objects.all()
        for related_resource_record in related_resource_records:
            se.index_data(index='resource_relations', doc_type='all', body=model_to_dict(related_resource_record), idfield='resourcexid')

    return result_summary

 def delete_index(self):   
     se = SearchEngineFactory().create() 
     query = Query(se, start=0, limit=10000)
     phrase = Match(field='conceptid', query=self.conceptid, type='phrase')
     query.add_query(phrase)
     query.delete(index='concept_labels')  
     se.delete_terms(self.id)

 def save(self):
     from arches.app.search.search_engine_factory import SearchEngineFactory
     se = SearchEngineFactory().create()
     if not self.created:
         self.created = datetime.datetime.now()
     self.modified = datetime.datetime.now()
     document = model_to_dict(self)
     se.index_data(index='resource_relations', doc_type='all', body=document, idfield='resourcexid')
     super(ResourceXResource, self).save()