Python Query.add_filter方法代码示例

# 需要导入模块: from arches.app.search.elasticsearch_dsl_builder import Query [as 别名]
# 或者: from arches.app.search.elasticsearch_dsl_builder.Query import add_filter [as 别名]
def get_related_resources(resourceid, lang, limit=1000, start=0):
    ret = {
        'resource_relationships': [],
        'related_resources': []
    }
    se = SearchEngineFactory().create()

    query = Query(se, limit=limit, start=start)
    query.add_filter(Terms(field='entityid1', terms=resourceid).dsl, operator='or')
    query.add_filter(Terms(field='entityid2', terms=resourceid).dsl, operator='or')
    resource_relations = query.search(index='resource_relations', doc_type='all')
    ret['total'] = resource_relations['hits']['total']

    entityids = set()
    for relation in resource_relations['hits']['hits']:
        relation['_source']['preflabel'] = get_preflabel_from_valueid(relation['_source']['relationshiptype'], lang)
        ret['resource_relationships'].append(relation['_source'])
        entityids.add(relation['_source']['entityid1'])
        entityids.add(relation['_source']['entityid2'])
    if len(entityids) > 0:
        entityids.remove(resourceid)   

    related_resources = se.search(index='entity', doc_type='_all', id=list(entityids))
    if related_resources:
        for resource in related_resources['docs']:
            ret['related_resources'].append(resource['_source'])

    return ret

# 需要导入模块: from arches.app.search.elasticsearch_dsl_builder import Query [as 别名]
# 或者: from arches.app.search.elasticsearch_dsl_builder.Query import add_filter [as 别名]
def get_preflabel_from_conceptid(conceptid, lang):
    ret = None
    default = {
        "category": "",
        "conceptid": "",
        "language": "",
        "value": "",
        "type": "",
        "id": ""
    }
    se = SearchEngineFactory().create()
    query = Query(se)
    terms = Terms(field='conceptid', terms=[conceptid])
    match = Match(field='type', query='preflabel', type='phrase')
    query.add_filter(terms)
    query.add_query(match)
    preflabels = query.search(index='concept_labels')['hits']['hits']
    for preflabel in preflabels:
        default = preflabel['_source']
        # get the label in the preferred language, otherwise get the label in the default language
        if preflabel['_source']['language'] == lang:
            return preflabel['_source']
        if preflabel['_source']['language'].split('-')[0] == lang.split('-')[0]:
            ret = preflabel['_source']
        if preflabel['_source']['language'] == settings.LANGUAGE_CODE and ret == None:
            ret = preflabel['_source']
    return default if ret == None else ret

# 需要导入模块: from arches.app.search.elasticsearch_dsl_builder import Query [as 别名]
# 或者: from arches.app.search.elasticsearch_dsl_builder.Query import add_filter [as 别名]
def get_preflabel_from_conceptid(conceptid, lang):
    ret = None
    default = {
        "category": "",
        "conceptid": "",
        "language": "",
        "value": "",
        "type": "",
        "id": ""
    }
    se = SearchEngineFactory().create()
    query = Query(se)
    terms = Terms(field='conceptid', terms=[conceptid])
    # Uncomment the following line only after having reindexed ElasticSearch cause currently the Arabic labels are indexed as altLabels
#     match = Match(field='type', query='prefLabel', type='phrase')
    query.add_filter(terms)
    # Uncomment the following line only after having reindexed ElasticSearch cause currently the Arabic labels are indexed as altLabels
#     query.add_query(match)

    preflabels = query.search(index='concept_labels')['hits']['hits'] 
    for preflabel in preflabels:
#         print 'Language at this point %s and label language %s and ret is %s' % (lang, preflabel['_source']['language'], ret)
        default = preflabel['_source']
        # get the label in the preferred language, otherwise get the label in the default language
        if preflabel['_source']['language'] == lang:
#             print 'prefLabel from Conceptid: %s' % preflabel['_source']
            return preflabel['_source']
        if preflabel['_source']['language'].split('-')[0] == lang.split('-')[0]:
            ret = preflabel['_source']
        if preflabel['_source']['language'] == lang and ret == None:
            ret = preflabel['_source']
    return default if ret == None else ret

# 需要导入模块: from arches.app.search.elasticsearch_dsl_builder import Query [as 别名]
# 或者: from arches.app.search.elasticsearch_dsl_builder.Query import add_filter [as 别名]
def get_related_resources(resourceid, lang, limit=1000, start=0, allowedtypes=[], is_anon=False):

    ret = {
        'resource_relationships': [],
        'related_resources': []
    }
    se = SearchEngineFactory().create()

    query = Query(se, limit=limit, start=start)
    query.add_filter(Terms(field='entityid1', terms=resourceid).dsl, operator='or')
    query.add_filter(Terms(field='entityid2', terms=resourceid).dsl, operator='or')
    resource_relations = query.search(index='resource_relations', doc_type="all")

    entityids = set()
    for relation in resource_relations['hits']['hits']:
        relation['_source']['preflabel'] = get_preflabel_from_valueid(relation['_source']['relationshiptype'], lang)
        ret['resource_relationships'].append(relation['_source'])
        entityids.add(relation['_source']['entityid1'])
        entityids.add(relation['_source']['entityid2'])
    if len(entityids) > 0:
        entityids.remove(resourceid)

    # can't figure why passing allowed types to doc_type param doesn't work,
    # so filter is carried out later
    related_resources = se.search(index='entity', doc_type='_all', id=list(entityids))

    filtered_ids = []
    if related_resources:
        for resource in related_resources['docs']:
            if not resource['_type'] in allowedtypes:
                filtered_ids.append(resource['_source']['entityid'])
                continue
            
            if is_anon:
                # filter out protected resources if user is anonymous
                # (this is basically a subset of the get_protected_entityids below
                # they should be combined probably)
                from search import get_protection_conceptids
                protect_id = get_protection_conceptids(settings.PROTECTION_LEVEL_NODE)
                conceptids = [d['conceptid'] for d in resource['_source']['domains']]
                if protect_id in conceptids:
                    filtered_ids.append(resource['_source']['entityid'])
                    continue
            ret['related_resources'].append(resource['_source'])
    
    if len(filtered_ids) > 0:
        # remove all relationships in ret that match a filtered id (this lc is yuge but I think concise)
        filtered_relationships = [rel for rel in ret['resource_relationships'] if not rel['entityid1'] in filtered_ids and not rel['entityid2'] in filtered_ids]
        
        # update ret values
        ret['resource_relationships'] = filtered_relationships
        
    ret['total'] = len(ret['resource_relationships'])
    
    return ret

# 需要导入模块: from arches.app.search.elasticsearch_dsl_builder import Query [as 别名]
# 或者: from arches.app.search.elasticsearch_dsl_builder.Query import add_filter [as 别名]
def get_related_resources(resourceid, lang='en-US', limit=1000, start=0):
    ret = {
        'resource_relationships': [],
        'related_resources': []
    }
    se = SearchEngineFactory().create()

    query = Query(se, limit=limit, start=start)
    query.add_filter(Terms(field='entityid1', terms=resourceid).dsl, operator='or')
    resource_relations = query.search(index='resource_relations', doc_type='all')
    ret['total'] = resource_relations['hits']['total']
    for relation in resource_relations['hits']['hits']:
        ret['resource_relationships'].append(relation['_source'])
    return ret

# 需要导入模块: from arches.app.search.elasticsearch_dsl_builder import Query [as 别名]
# 或者: from arches.app.search.elasticsearch_dsl_builder.Query import add_filter [as 别名]
def get_preflabel_from_conceptid(conceptid, lang):
    ret = None
    default = {"category": "", "conceptid": "", "language": "", "value": "", "type": "", "id": ""}
    se = SearchEngineFactory().create()
    query = Query(se)
    terms = Terms(field="conceptid", terms=[conceptid])
    match = Match(field="type", query="preflabel", type="phrase")
    query.add_filter(terms)
    query.add_query(match)
    preflabels = query.search(index="concept_labels")["hits"]["hits"]
    for preflabel in preflabels:
        default = preflabel["_source"]
        # get the label in the preferred language, otherwise get the label in the default language
        if preflabel["_source"]["language"] == lang:
            return preflabel["_source"]
        if preflabel["_source"]["language"].split("-")[0] == lang.split("-")[0]:
            ret = preflabel["_source"]
        if preflabel["_source"]["language"] == settings.LANGUAGE_CODE and ret == None:
            ret = preflabel["_source"]
    return default if ret == None else ret

# 需要导入模块: from arches.app.search.elasticsearch_dsl_builder import Query [as 别名]
# 或者: from arches.app.search.elasticsearch_dsl_builder.Query import add_filter [as 别名]
def get_related_resource_ids(resourceids, lang, limit=1000, start=0):
    se = SearchEngineFactory().create()
    query = Query(se, limit=limit, start=start)
    query.add_filter(Terms(field='entityid1', terms=resourceids).dsl, operator='or')
    query.add_filter(Terms(field='entityid2', terms=resourceids).dsl, operator='or')
    resource_relations = query.search(  index='resource_relations', doc_type='all')
    
    entityids = set()
    for relation in resource_relations['hits']['hits']: 
        # add the other halves add the relations which are not in the original list of ids
        from_is_original_result = relation['_source']['entityid1'] in resourceids
        to_is_original_result = relation['_source']['entityid2'] in resourceids
        
        if from_is_original_result:
            entityids.add(relation['_source']['entityid2'])
            
        if to_is_original_result:
            entityids.add(relation['_source']['entityid1'])
    
    return entityids

# 需要导入模块: from arches.app.search.elasticsearch_dsl_builder import Query [as 别名]
# 或者: from arches.app.search.elasticsearch_dsl_builder.Query import add_filter [as 别名]

#.........这里部分代码省略.........
    se = SearchEngineFactory().create()

    if export != None:
        limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE  
    else:
        limit = settings.SEARCH_ITEMS_PER_PAGE
    
    query = Query(se, start=limit*int(page-1), limit=limit)
    boolquery = Bool()
    boolfilter = Bool()
    
    if term_filter != '':
        # Ce uporabnik ni avtenticiran, prikazemo le veljavne (to je verjetno potrebno se dodelati (mogoce da vidijo le svoje???)!!!)
        if (request.user.username == 'anonymous'):
            auto_filter = []
            for item in JSONDeserializer().deserialize(term_filter):
               auto_filter.append(item) 
            
            # Poiscimo concept id in context za Published status
            AUTO_TERM_FILTER = get_auto_filter(request)
            
            auto_filter.append(AUTO_TERM_FILTER)
            term_filter = JSONSerializer().serialize(auto_filter)
            
    print 'term_filter'
    if term_filter != '':
        for term in JSONDeserializer().deserialize(term_filter):
            print term
            if term['type'] == 'term':
                entitytype = models.EntityTypes.objects.get(conceptid_id=term['context'])
                boolfilter_nested = Bool()
                boolfilter_nested.must(Terms(field='child_entities.entitytypeid', terms=[entitytype.pk]))
                boolfilter_nested.must(Match(field='child_entities.value', query=term['value'], type='phrase'))
                nested = Nested(path='child_entities', query=boolfilter_nested)
                if term['inverted']:
                    boolfilter.must_not(nested)
                else:    
                    boolfilter.must(nested)
            elif term['type'] == 'concept':
                concept_ids = _get_child_concepts(term['value'])
                terms = Terms(field='domains.conceptid', terms=concept_ids)
                nested = Nested(path='domains', query=terms)
                if term['inverted']:
                    boolfilter.must_not(nested)
                else:
                    boolfilter.must(nested)
            elif term['type'] == 'string':
                boolfilter_folded = Bool()
                boolfilter_folded.should(Match(field='child_entities.value', query=term['value'], type='phrase_prefix'))
                boolfilter_folded.should(Match(field='child_entities.value.folded', query=term['value'], type='phrase_prefix'))
                nested = Nested(path='child_entities', query=boolfilter_folded)
                if term['inverted']:
                    boolquery.must_not(nested)
                else:    
                    boolquery.must(nested)
    if 'geometry' in spatial_filter and 'type' in spatial_filter['geometry'] and spatial_filter['geometry']['type'] != '':
        geojson = spatial_filter['geometry']
        if geojson['type'] == 'bbox':
            coordinates = [[geojson['coordinates'][0],geojson['coordinates'][3]], [geojson['coordinates'][2],geojson['coordinates'][1]]]
            geoshape = GeoShape(field='geometries.value', type='envelope', coordinates=coordinates )
            nested = Nested(path='geometries', query=geoshape)
        else:
            buffer = spatial_filter['buffer']
            geojson = JSONDeserializer().deserialize(_buffer(geojson,buffer['width'],buffer['unit']).json)
            geoshape = GeoShape(field='geometries.value', type=geojson['type'], coordinates=geojson['coordinates'] )
            nested = Nested(path='geometries', query=geoshape)

        if 'inverted' not in spatial_filter:
            spatial_filter['inverted'] = False

        if spatial_filter['inverted']:
            boolfilter.must_not(nested)
        else:
            boolfilter.must(nested)

    if 'year_min_max' in temporal_filter and len(temporal_filter['year_min_max']) == 2:
        start_date = date(temporal_filter['year_min_max'][0], 1, 1)
        end_date = date(temporal_filter['year_min_max'][1], 12, 31)
        if start_date:
            start_date = start_date.isoformat()
        if end_date:
            end_date = end_date.isoformat()
        range = Range(field='dates.value', gte=start_date, lte=end_date)
        nested = Nested(path='dates', query=range)
        
        if 'inverted' not in temporal_filter:
            temporal_filter['inverted'] = False

        if temporal_filter['inverted']:
            boolfilter.must_not(nested)
        else:
            boolfilter.must(nested)
        
    if not boolquery.empty:
        query.add_query(boolquery)

    if not boolfilter.empty:
        query.add_filter(boolfilter)

    return query

# 需要导入模块: from arches.app.search.elasticsearch_dsl_builder import Query [as 别名]
# 或者: from arches.app.search.elasticsearch_dsl_builder.Query import add_filter [as 别名]

#.........这里部分代码省略.........

                        if date_operator == '1': # equals query
                            range = Range(field='dates.value', gte=date_value, lte=date_value)
                        elif date_operator == '0': # greater than query 
                            range = Range(field='dates.value', lt=date_value)
                        elif date_operator == '2': # less than query
                            range = Range(field='dates.value', gt=date_value)
                        
                        nested = Nested(path='dates', query=range)
                        if 'inverted' not in temporal_filter[index]:
                            temporal_filter[index]['inverted'] = False

                        if temporal_filter[index]['inverted']:
                            selectbox_boolfilter.must_not(nested)
                        else:
                            selectbox_boolfilter.must(nested)


            terms_queries.append(selectbox_boolfilter)
            # if not selectbox_boolfilter.empty:
            #     if boolean_search == 'or':
            #         boolfilter.should(selectbox_boolfilter)
            #     else:
            #         boolfilter.must(selectbox_boolfilter)
        
        # We now have individual query terms for each of the search components. Combine into one group now
        # Start by building a an array of groups which will be combined according to the global And/Or
        # Queries within one of these groups will be combined by the complement of the global And/Or
        # We may end up with [ [A,B], [C], [D,E] ], which would translate to either:
        #    (A || B) && C && (D || E)
        #       or
        #    (A && B) || C || (D && E)
        # for global AND or OR respectively
        
        # logging.warning("TERMS QUERIES %s", terms_queries)
        
        bool_components = [];
        
        for i, term_query in enumerate(terms_queries):
            if i is 0:
                bool_components.append([term_query])
            else:
                should_group_with_previous = filter_combine_flags[i-1]
                if should_group_with_previous:
                    bool_components[-1].append(term_query)
                else:
                    bool_components.append([term_query])
            
        # logging.warning("BOOL COMPONENTS %s", bool_components)
        # Now build the ES queries
        for bool_component in bool_components:
            if len(bool_component) is 1:
                # just combine this on its own
                q = bool_component[0]
            else:
                q = Bool()
                for sub_component in bool_component:
                    if boolean_search == 'or':
                        #apply the OPPOSITE of the global boolean operator
                        q.must(sub_component)
                    else:
                        q.should(sub_component)
                        
            # combine to the overall query according to the global boolean operator
            if boolean_search == 'or':
                boolfilter.should(q)
            else:
                boolfilter.must(q)

    if 'geometry' in spatial_filter and 'type' in spatial_filter['geometry'] and spatial_filter['geometry']['type'] != '':
        geojson = spatial_filter['geometry']
        if geojson['type'] == 'bbox':
            coordinates = [[geojson['coordinates'][0],geojson['coordinates'][3]], [geojson['coordinates'][2],geojson['coordinates'][1]]]
            geoshape = GeoShape(field='geometries.value', type='envelope', coordinates=coordinates )
            nested = Nested(path='geometries', query=geoshape)
        else:
            buffer = spatial_filter['buffer']
            geojson = JSONDeserializer().deserialize(_buffer(geojson,buffer['width'],buffer['unit']).json)
            geoshape = GeoShape(field='geometries.value', type=geojson['type'], coordinates=geojson['coordinates'] )
            nested = Nested(path='geometries', query=geoshape)

        if 'inverted' not in spatial_filter:
            spatial_filter['inverted'] = False

        if spatial_filter['inverted']:
            boolfilter.must_not(nested)
        else:
            boolfilter.must(nested)

    if not boolquery.empty:
        query.add_query(boolquery)

    if not boolfilter.empty:
        query.add_filter(boolfilter)
    
#  Sorting criterion added to query (AZ 10/08/16)
    query.dsl.update({'sort': sorting})
    # logging.warning("-=-==-=-===-=--=-==-=-===-=- query: -=-==-=-===-=--=-==-=-===-=-> %s", query)

    return query

# 需要导入模块: from arches.app.search.elasticsearch_dsl_builder import Query [as 别名]
# 或者: from arches.app.search.elasticsearch_dsl_builder.Query import add_filter [as 别名]
def build_search_results_dsl(request):
    term_filter = request.GET.get('termFilter', '')
    spatial_filter = JSONDeserializer().deserialize(request.GET.get('spatialFilter', None)) 
    export = request.GET.get('export', None)
    page = 1 if request.GET.get('page') == '' else int(request.GET.get('page', 1))
    temporal_filter = JSONDeserializer().deserialize(request.GET.get('temporalFilter', None))

    se = SearchEngineFactory().create()

    if export != None:
        limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE  
    else:
        limit = settings.SEARCH_ITEMS_PER_PAGE
    
    query = Query(se, start=limit*int(page-1), limit=limit)
    boolquery = Bool()
    boolfilter = Bool()
    
    if term_filter != '':
        for term in JSONDeserializer().deserialize(term_filter):
            if term['type'] == 'term':
                entitytype = models.EntityTypes.objects.get(conceptid_id=term['context'])
                boolfilter_nested = Bool()
                boolfilter_nested.must(Terms(field='child_entities.entitytypeid', terms=[entitytype.pk]))
                boolfilter_nested.must(Match(field='child_entities.value', query=term['value'], type='phrase'))
                nested = Nested(path='child_entities', query=boolfilter_nested)
                if term['inverted']:
                    boolfilter.must_not(nested)
                else:    
                    boolfilter.must(nested)
            elif term['type'] == 'concept':
                concept_ids = _get_child_concepts(term['value'])
                terms = Terms(field='domains.conceptid', terms=concept_ids)
                nested = Nested(path='domains', query=terms)
                if term['inverted']:
                    boolfilter.must_not(nested)
                else:
                    boolfilter.must(nested)
            elif term['type'] == 'string':
                boolfilter_folded = Bool()
                boolfilter_folded.should(Match(field='child_entities.value', query=term['value'], type='phrase_prefix'))
                boolfilter_folded.should(Match(field='child_entities.value.folded', query=term['value'], type='phrase_prefix'))
                nested = Nested(path='child_entities', query=boolfilter_folded)
                if term['inverted']:
                    boolquery.must_not(nested)
                else:    
                    boolquery.must(nested)

    if 'geometry' in spatial_filter and 'type' in spatial_filter['geometry'] and spatial_filter['geometry']['type'] != '':
        geojson = spatial_filter['geometry']
        if geojson['type'] == 'bbox':
            coordinates = [[geojson['coordinates'][0],geojson['coordinates'][3]], [geojson['coordinates'][2],geojson['coordinates'][1]]]
            geoshape = GeoShape(field='geometries.value', type='envelope', coordinates=coordinates )
            nested = Nested(path='geometries', query=geoshape)
        else:
            buffer = spatial_filter['buffer']
            geojson = JSONDeserializer().deserialize(_buffer(geojson,buffer['width'],buffer['unit']).json)
            geoshape = GeoShape(field='geometries.value', type=geojson['type'], coordinates=geojson['coordinates'] )
            nested = Nested(path='geometries', query=geoshape)

        if 'inverted' not in spatial_filter:
            spatial_filter['inverted'] = False

        if spatial_filter['inverted']:
            boolfilter.must_not(nested)
        else:
            boolfilter.must(nested)

    if 'year_min_max' in temporal_filter and len(temporal_filter['year_min_max']) == 2:
        start_date = date(temporal_filter['year_min_max'][0], 1, 1)
        end_date = date(temporal_filter['year_min_max'][1], 12, 31)
        if start_date:
            start_date = start_date.isoformat()
        if end_date:
            end_date = end_date.isoformat()
        range = Range(field='dates.value', gte=start_date, lte=end_date)
        nested = Nested(path='dates', query=range)
        
        if 'inverted' not in temporal_filter:
            temporal_filter['inverted'] = False

        if temporal_filter['inverted']:
            boolfilter.must_not(nested)
        else:
            boolfilter.must(nested)
        
    if not boolquery.empty:
        query.add_query(boolquery)

    if not boolfilter.empty:
        query.add_filter(boolfilter)

    return query