Python elasticsearch_dsl_builder.Query类代码示例

 def delete_concept_values_index(concepts_to_delete):
     se = SearchEngineFactory().create()
     for concept in concepts_to_delete.itervalues():
         query = Query(se, start=0, limit=10000)
         term = Term(field='conceptid', term=concept.id)
         query.add_query(term)
         query.delete(index='strings', doc_type='concept')

def get_preflabel_from_conceptid(conceptid, lang):
    ret = None
    default = {
        "category": "",
        "conceptid": "",
        "language": "",
        "value": "",
        "type": "",
        "id": ""
    }
    se = SearchEngineFactory().create()
    query = Query(se)
    terms = Terms(field='conceptid', terms=[conceptid])
    # Uncomment the following line only after having reindexed ElasticSearch cause currently the Arabic labels are indexed as altLabels
#     match = Match(field='type', query='prefLabel', type='phrase')
    query.add_filter(terms)
    # Uncomment the following line only after having reindexed ElasticSearch cause currently the Arabic labels are indexed as altLabels
#     query.add_query(match)

    preflabels = query.search(index='concept_labels')['hits']['hits'] 
    for preflabel in preflabels:
#         print 'Language at this point %s and label language %s and ret is %s' % (lang, preflabel['_source']['language'], ret)
        default = preflabel['_source']
        # get the label in the preferred language, otherwise get the label in the default language
        if preflabel['_source']['language'] == lang:
#             print 'prefLabel from Conceptid: %s' % preflabel['_source']
            return preflabel['_source']
        if preflabel['_source']['language'].split('-')[0] == lang.split('-')[0]:
            ret = preflabel['_source']
        if preflabel['_source']['language'] == lang and ret == None:
            ret = preflabel['_source']
    return default if ret == None else ret

def arch_investigation_layer(request, boundtype=''):

    data = []
    geom_param = request.GET.get('geom', None)

    bbox = request.GET.get('bbox', '')
    limit = request.GET.get('limit', settings.MAP_LAYER_FEATURE_LIMIT)
    geojson_collection = {
      "type": "FeatureCollection",
      "features": []
    }

    se = SearchEngineFactory().create()
    query = Query(se, limit=limit)

    args = {
        'index':'entity',
        'doc_type':'ARCHAEOLOGICAL_ZONE.E53',
    }

    data = query.search(**args)
    for item in data['hits']['hits']:
        for geom in item['_source']['geometries']:
            if geom['entitytypeid'] == 'SHOVEL_TEST_GEOMETRY.E47':
                print json.dumps(geom,indent=2)
                feat = {
                    'geometry':geom['value'],
                    'type':"Feature",
                    'id':item['_source']['entityid'],
                    }
                geojson_collection['features'].append(feat)

    return JSONResponse(geojson_collection)

def get_preflabel_from_conceptid(conceptid, lang):
    ret = None
    default = {
        "category": "",
        "conceptid": "",
        "language": "",
        "value": "",
        "type": "",
        "id": ""
    }
    se = SearchEngineFactory().create()
    query = Query(se)
    bool_query = Bool()
    bool_query.must(Match(field='type', query='prefLabel', type='phrase'))
    bool_query.filter(Terms(field='conceptid', terms=[conceptid]))
    query.add_query(bool_query)
    preflabels = query.search(index='strings', doc_type='concept')['hits']['hits']
    for preflabel in preflabels:
        default = preflabel['_source']
        # get the label in the preferred language, otherwise get the label in the default language
        if preflabel['_source']['language'] == lang:
            return preflabel['_source']
        if preflabel['_source']['language'].split('-')[0] == lang.split('-')[0]:
            ret = preflabel['_source']
        if preflabel['_source']['language'] == settings.LANGUAGE_CODE and ret == None:
            ret = preflabel['_source']
    return default if ret == None else ret

    def test_bulk_delete(self):
        """
        Test bulk deleting of documents in Elasticsearch

        """

        se = SearchEngineFactory().create()
        # se.create_index(index='test')

        for i in range(10):
            x = {
                'id': i,
                'type': 'prefLabel',
                'value': 'test pref label',
            }
            se.index_data(index='test', doc_type='test', body=x, idfield='id', refresh=True)
            y = {
                'id': i + 100,
                'type': 'altLabel',
                'value': 'test alt label',
            }
            se.index_data(index='test', doc_type='test', body=y, idfield='id', refresh=True)


        query = Query(se, start=0, limit=100)
        match = Match(field='type', query='altLabel')
        query.add_query(match)

        query.delete(index='test', refresh=True)

        self.assertEqual(se.es.count(index='test', doc_type='test')['count'], 10)

def get_related_resources(resourceid, lang, limit=1000, start=0):
    ret = {
        'resource_relationships': [],
        'related_resources': []
    }
    se = SearchEngineFactory().create()

    query = Query(se, limit=limit, start=start)
    query.add_filter(Terms(field='entityid1', terms=resourceid).dsl, operator='or')
    query.add_filter(Terms(field='entityid2', terms=resourceid).dsl, operator='or')
    resource_relations = query.search(index='resource_relations', doc_type='all')
    ret['total'] = resource_relations['hits']['total']

    entityids = set()
    for relation in resource_relations['hits']['hits']:
        relation['_source']['preflabel'] = get_preflabel_from_valueid(relation['_source']['relationshiptype'], lang)
        ret['resource_relationships'].append(relation['_source'])
        entityids.add(relation['_source']['entityid1'])
        entityids.add(relation['_source']['entityid2'])
    if len(entityids) > 0:
        entityids.remove(resourceid)   

    related_resources = se.search(index='entity', doc_type='_all', id=list(entityids))
    if related_resources:
        for resource in related_resources['docs']:
            ret['related_resources'].append(resource['_source'])

    return ret

 def get_relations(resourceinstanceid, start, limit):
     query = Query(se, start=start, limit=limit)
     bool_filter = Bool()
     bool_filter.should(Terms(field='resourceinstanceidfrom', terms=resourceinstanceid))
     bool_filter.should(Terms(field='resourceinstanceidto', terms=resourceinstanceid))
     query.add_query(bool_filter)
     return query.search(index='resource_relations', doc_type='all')

def get_auto_filter(request):
    lang = request.GET.get('lang', settings.LANGUAGE_CODE)
    se1 = SearchEngineFactory().create()
    searchString1 = settings.PUBLISHED_LABEL
    query1 = Query(se1, start=0, limit=settings.SEARCH_DROPDOWN_LENGTH)
    boolquery1 = Bool()
    boolquery1.should(Match(field='term', query=searchString1.lower(), type='phrase_prefix', fuzziness='AUTO'))
    boolquery1.should(Match(field='term.folded', query=searchString1.lower(), type='phrase_prefix', fuzziness='AUTO'))
    boolquery1.should(Match(field='term.folded', query=searchString1.lower(), fuzziness='AUTO'))
    query1.add_query(boolquery1)
    results1 = query1.search(index='term', doc_type='value')
    conceptid1 = ''
    context1 = ''
    for result1 in results1['hits']['hits']:
        prefLabel = get_preflabel_from_conceptid(result1['_source']['context'], lang)
        result1['_source']['options']['context_label'] = prefLabel['value']
        if (prefLabel['value'] == settings.EW_STATUS_TERM and result1['_source']['term'] == settings.PUBLISHED_LABEL)  :
            conceptid1 = result1['_source']['options']['conceptid']
            context1 = result1['_source']['context']
    AUTO_TERM_FILTER = {"inverted": False, "type": "concept"}
    AUTO_TERM_FILTER["text"] = settings.PUBLISHED_LABEL
    AUTO_TERM_FILTER["value"] = conceptid1
    AUTO_TERM_FILTER["context"] = context1
    AUTO_TERM_FILTER["context_label"] = settings.EW_STATUS_TERM
    AUTO_TERM_FILTER["id"] = AUTO_TERM_FILTER['text'] + conceptid1
    return AUTO_TERM_FILTER

    def get_related_resources(self, lang='en-US', limit=1000, start=0):
        """
        Returns an object that lists the related resources, the relationship types, and a reference to the current resource

        """

        ret = {
            'resource_instance': self,
            'resource_relationships': [],
            'related_resources': []
        }
        se = SearchEngineFactory().create()
        query = Query(se, limit=limit, start=start)
        bool_filter = Bool()
        bool_filter.should(Terms(field='resourceinstanceidfrom', terms=self.resourceinstanceid))
        bool_filter.should(Terms(field='resourceinstanceidto', terms=self.resourceinstanceid))
        query.add_query(bool_filter)
        resource_relations = query.search(index='resource_relations', doc_type='all')
        ret['total'] = resource_relations['hits']['total']
        instanceids = set()
        for relation in resource_relations['hits']['hits']:
            relation['_source']['preflabel'] = get_preflabel_from_valueid(relation['_source']['relationshiptype'], lang)
            ret['resource_relationships'].append(relation['_source'])
            instanceids.add(relation['_source']['resourceinstanceidto'])
            instanceids.add(relation['_source']['resourceinstanceidfrom'])
        if len(instanceids) > 0:
            instanceids.remove(str(self.resourceinstanceid))

        related_resources = se.search(index='resource', doc_type='_all', id=list(instanceids))
        if related_resources:
            for resource in related_resources['docs']:
                ret['related_resources'].append(resource['_source'])

        return ret

def reverse_func(apps, schema_editor):
    extensions = [os.path.join(settings.ONTOLOGY_PATH, x) for x in settings.ONTOLOGY_EXT]
    management.call_command('load_ontology', source=os.path.join(settings.ONTOLOGY_PATH, settings.ONTOLOGY_BASE),
        version=settings.ONTOLOGY_BASE_VERSION, ontology_name=settings.ONTOLOGY_BASE_NAME, id=settings.ONTOLOGY_BASE_ID, extensions=','.join(extensions), verbosity=0)

    Node = apps.get_model("models", "Node")
    Edge = apps.get_model("models", "Edge")

    for node in Node.objects.all():
        node.ontologyclass = str(node.ontologyclass).split('/')[-1]
        node.save()

    for edge in Edge.objects.all():
        edge.ontologyproperty = str(edge.ontologyproperty).split('/')[-1]
        edge.save()

    # remove index for base Arches concept
    se = SearchEngineFactory().create()
    query = Query(se, start=0, limit=10000)
    query.add_query(Term(field='conceptid', term='00000000-0000-0000-0000-000000000001'))
    query.delete(index='strings', doc_type='concept')

    try:
        DValueType = apps.get_model("models", "DValueType")
        DValueType.objects.get(valuetype='identifier').delete()
    except:
        pass

 def delete_index(self):   
     se = SearchEngineFactory().create() 
     query = Query(se, start=0, limit=10000)
     phrase = Match(field='conceptid', query=self.conceptid, type='phrase')
     query.add_query(phrase)
     query.delete(index='concept_labels')  
     se.delete_terms(self.id)

def build_search_terms_dsl(request):
    se = SearchEngineFactory().create()
    searchString = request.GET.get('q', '')
    query = Query(se, start=0, limit=settings.SEARCH_DROPDOWN_LENGTH)
    boolquery = Bool()
    boolquery.should(Match(field='term', query=searchString.lower(), type='phrase_prefix', fuzziness='AUTO'))
    boolquery.should(Match(field='term.folded', query=searchString.lower(), type='phrase_prefix', fuzziness='AUTO'))
    boolquery.should(Match(field='term.folded', query=searchString.lower(), fuzziness='AUTO'))
    query.add_query(boolquery)
    return query

def get_related_resources(resourceid, lang, limit=1000, start=0, allowedtypes=[], is_anon=False):

    ret = {
        'resource_relationships': [],
        'related_resources': []
    }
    se = SearchEngineFactory().create()

    query = Query(se, limit=limit, start=start)
    query.add_filter(Terms(field='entityid1', terms=resourceid).dsl, operator='or')
    query.add_filter(Terms(field='entityid2', terms=resourceid).dsl, operator='or')
    resource_relations = query.search(index='resource_relations', doc_type="all")

    entityids = set()
    for relation in resource_relations['hits']['hits']:
        relation['_source']['preflabel'] = get_preflabel_from_valueid(relation['_source']['relationshiptype'], lang)
        ret['resource_relationships'].append(relation['_source'])
        entityids.add(relation['_source']['entityid1'])
        entityids.add(relation['_source']['entityid2'])
    if len(entityids) > 0:
        entityids.remove(resourceid)

    # can't figure why passing allowed types to doc_type param doesn't work,
    # so filter is carried out later
    related_resources = se.search(index='entity', doc_type='_all', id=list(entityids))

    filtered_ids = []
    if related_resources:
        for resource in related_resources['docs']:
            if not resource['_type'] in allowedtypes:
                filtered_ids.append(resource['_source']['entityid'])
                continue
            
            if is_anon:
                # filter out protected resources if user is anonymous
                # (this is basically a subset of the get_protected_entityids below
                # they should be combined probably)
                from search import get_protection_conceptids
                protect_id = get_protection_conceptids(settings.PROTECTION_LEVEL_NODE)
                conceptids = [d['conceptid'] for d in resource['_source']['domains']]
                if protect_id in conceptids:
                    filtered_ids.append(resource['_source']['entityid'])
                    continue
            ret['related_resources'].append(resource['_source'])
    
    if len(filtered_ids) > 0:
        # remove all relationships in ret that match a filtered id (this lc is yuge but I think concise)
        filtered_relationships = [rel for rel in ret['resource_relationships'] if not rel['entityid1'] in filtered_ids and not rel['entityid2'] in filtered_ids]
        
        # update ret values
        ret['resource_relationships'] = filtered_relationships
        
    ret['total'] = len(ret['resource_relationships'])
    
    return ret

def map_layers(request, entitytypeid='all', get_centroids=False):
    data = []
    geom_param = request.GET.get('geom', None)

    bbox = request.GET.get('bbox', '')
    limit = request.GET.get('limit', settings.MAP_LAYER_FEATURE_LIMIT)
    entityids = request.GET.get('entityid', '')
    geojson_collection = {
      "type": "FeatureCollection",
      "features": []
    }
    
    se = SearchEngineFactory().create()
    query = Query(se, limit=limit)

    args = { 'index': 'maplayers' }
    if entitytypeid != 'all':
        args['doc_type'] = entitytypeid
    if entityids != '':
        for entityid in entityids.split(','):
            geojson_collection['features'].append(se.search(index='maplayers', id=entityid)['_source'])
        return JSONResponse(geojson_collection)

    data = query.search(**args)
    if not data:
        return JSONResponse({})
    for item in data['hits']['hits']:
        # Ce uporabnik ni avtenticiran, prikazemo le veljavne (to je verjetno potrebno se dodelati (mogoce da vidijo le svoje???)!!!)
        if (not request.user.username != 'anonymous'):
            if (item['_source']['properties']['ewstatus'] != settings.PUBLISHED_LABEL):
                continue
        if get_centroids:
            item['_source']['geometry'] = item['_source']['properties']['centroid']
            #item['_source'].pop('properties', None)
            item['_source']['properties'].pop('extent', None)
            item['_source']['properties'].pop('elements', None)
            item['_source']['properties'].pop('entitytypeid', None)
            item['_source']['properties'].pop('constructions', None)
            item['_source']['properties'].pop('centroid', None)
            item['_source']['properties'].pop('ewstatus', None)
            item['_source']['properties'].pop('address', None)
            item['_source']['properties'].pop('designations', None)
            item['_source']['properties'].pop('primaryname', None)
            item['_source']['properties'].pop('resource_type', None)
        elif geom_param != None:
            item['_source']['geometry'] = item['_source']['properties'][geom_param]
            item['_source']['properties'].pop('extent', None)
            item['_source']['properties'].pop(geom_param, None)
        else:
            item['_source']['properties'].pop('extent', None)
            item['_source']['properties'].pop('centroid', None)
        geojson_collection['features'].append(item['_source'])
    return JSONResponse(geojson_collection)  

def map_layers(request, entitytypeid='all', get_centroids=False):

    data = []

    geom_param = request.GET.get('geom', None)

    bbox = request.GET.get('bbox', '')
    limit = request.GET.get('limit', settings.MAP_LAYER_FEATURE_LIMIT)
    entityids = request.GET.get('entityid', '')
    geojson_collection = {
      "type": "FeatureCollection",
      "features": []
    }
    
    se = SearchEngineFactory().create()
    query = Query(se, limit=limit)

    args = { 'index': 'maplayers' }
    if entitytypeid != 'all':
        args['doc_type'] = entitytypeid
    if entityids != '':
        for entityid in entityids.split(','):
            geojson_collection['features'].append(se.search(index='maplayers', id=entityid)['_source'])
        return JSONResponse(geojson_collection)

    data = query.search(**args)
    
    # if anonymous user, get list of protected entity ids to be excluded from map
    protected = []
    
    if request.user.username == 'anonymous':
        protected = get_protected_entityids()
        print protected

    for item in data['hits']['hits']:
        if item['_id'] in protected:
            print "hide this one"
            print json.dumps(item,indent=2)
            continue
        if get_centroids:
            item['_source']['geometry'] = item['_source']['properties']['centroid']
            item['_source'].pop('properties', None)
        elif geom_param != None:
            item['_source']['geometry'] = item['_source']['properties'][geom_param]
            item['_source']['properties'].pop('extent', None)
            item['_source']['properties'].pop(geom_param, None)
        else:
            item['_source']['properties'].pop('extent', None)
            item['_source']['properties'].pop('centroid', None)
        geojson_collection['features'].append(item['_source'])

    return JSONResponse(geojson_collection)