Python ES.search方法代码示例

# 需要导入模块: from pyes import ES [as 别名]
# 或者: from pyes.ES import search [as 别名]
class BaseElasticSearchClient(BaseClient):

    def __init__(self, servers, index):
        """
        @param servers: Make sure to include the port with the server address
        @param index: Document index
        @return:
        """
        super(BaseElasticSearchClient, self).__init__()
        self.connection = None
        self.servers = servers
        self.index = index if type(index) is list else [index]

    def connect(self, connection_pool=1):
        update_connection_pool(connection_pool)

        try:
            self.connection = ES(self.servers)
        except NoServerAvailable:
            self._log.error('Failed to connect to elastic search server')
            return False
        return True

    def close(self):
        self.connection = None

    def _create_term_query(self, must_list):
        # TODO: add remaining conditional list functionality.
        query = BoolQuery()
        for term in must_list:
            query.add_must(term)

    def find_term(self, name, value, size=10):
        if not self.connection:
            return

        query = TermQuery(name, value)
        return self.connection.search(query=Search(query, size=size),
                                      indices=self.index)

    def find(self, filter_terms, size=10, doc_types=None):
        if not self.connection:
            return

        query = self._create_term_query(must_list=filter_terms)
        return self.connection.search(query=Search(query, size=size),
                                      indices=self.index,
                                      doc_types=doc_types)

    def find_one(self, filter_terms, size=10, doc_types=None):
        if not self.connection:
            return

        results = self.find(filter_terms=filter_terms, size=size,
                            doc_types=doc_types)
        return results[0] if len(results) > 0 else None

# 需要导入模块: from pyes import ES [as 别名]
# 或者: from pyes.ES import search [as 别名]
def facets(host='localhost:9200',
          facet_terms=['bibleverse'],
          _type='habakkuk',
          date_filter=[],
          size=10):
    ret = {}
    conn = ES(host)
    q = MatchAllQuery()
    if date_filter:
        start,end = date_filter
        q = FilteredQuery(q, RangeFilter(qrange=ESRange('created_at_date',
                                                        start.isoformat(),
                                                        end.isoformat(),
                                                        include_upper=False)))

    q = q.search(size=0)
    for term in facet_terms:
        q.facet.add_term_facet(term,order='count',size=size)
        
    es_logger.info(q.serialize())

    resultset = conn.search(query=q, indices=_type+'-*', doc_types=[_type])
    for facet in resultset.facets:
        ret[facet] = []
        for row in resultset.facets[facet]['terms']:
            ret[facet].append({"value":row['term'],"count":row['count']})

    logger.debug("facets return|'%s'"%json.dumps(ret))
    return ret

# 需要导入模块: from pyes import ES [as 别名]
# 或者: from pyes.ES import search [as 别名]
def term_facet(host='localhost:9200',
               terms=['bibleverse'],
               _type='habakkuk',
               date_filter=[],
               size=10):
    ret = []
    conn = ES(host)
    q = MatchAllQuery()
    if date_filter:
        start,end = date_filter
        q = FilteredQuery(q, RangeFilter(qrange=ESRange('created_at_date',start,end,include_upper=False)))

    q = q.search(size=0)
    for term in terms:
        q.facet.add_term_facet(term,order='count',size=size)
        
    print json.dumps(json.loads(q.to_search_json()),indent=2)

    resultset = conn.search(query=q, indices=_type+'-*', doc_types=[_type])
    for facet in resultset.facets:
        print "Total",facet,resultset.facets[facet]['total']
        for row in resultset.facets[facet]['terms']:
            print "\t",row['term'],row['count']
            ret.append((facet,row['term']))
        
    return ret

# 需要导入模块: from pyes import ES [as 别名]
# 或者: from pyes.ES import search [as 别名]
def search_people_by_bio(query, limit_results=DEFAULT_LIMIT,
                         index=['onename_people_index']):
    """ queries lucene index to find a nearest match, output is profile username
    """

    from pyes import QueryStringQuery, ES
    conn = ES()

    q = QueryStringQuery(query,
                         search_fields=['username', 'profile_bio'],
                         default_operator='and')

    results = conn.search(query=q, size=20, indices=index)
    count = conn.count(query=q)
    count = count.count

    # having 'or' gives more results but results quality goes down
    if(count == 0):

        q = QueryStringQuery(query,
                             search_fields=['username', 'profile_bio'],
                             default_operator='or')

        results = conn.search(query=q, size=20, indices=index)

    results_list = []
    counter = 0

    for profile in results:

        username = profile['username']
        results_list.append(username)

        counter += 1

        if(counter == limit_results):
            break

    return results_list

# 需要导入模块: from pyes import ES [as 别名]
# 或者: from pyes.ES import search [as 别名]
class ESPages():
    ''' For use with Django's paginator. Currently not used after pyes
        update implemented ResultSet, which provides the count,
        __getitem__, and __len__ methods required for Django's paginator. '''
    def __init__(self, es_query, **kwargs):
        ''' Make initial ES query'''
        self.conn = ES(settings.ES_HOST[0], timeout=10.0)
        self.es_query = es_query
        res = self.conn.search(query=self.es_query, size='0', **kwargs)
        self.total_hits = res['hits']['total']

    def count(self):
        return self.total_hits

    def __getitem__(self, q_slice):
        ''' Make ES query for range of hits'''
        q = self.es_query.search(start=str(q_slice.start), size=str(q_slice.stop-q_slice.start+1))
        res = self.conn.search(q)
        return res['hits']['hits']

    def __len__(self):
        return self.count()

# 需要导入模块: from pyes import ES [as 别名]
# 或者: from pyes.ES import search [as 别名]
def search(searchkey=u"电影"):
    conn = ES('127.0.0.1:9200')
    # TextQuery会对searchkey进行分词
    qtitle = TextQuery("title", searchkey)
    h = HighLighter(['<b>'], ['</b>'], fragment_size=500)
    # 多字段搜索(must=>and,should=>or)，高亮，结果截取（分页），排序
    q = Search(BoolQuery(should=[qtitle]), highlight=h, start=0, size=3,
               sort={'id': {'order': 'asc'}})
    q.add_highlight("title")
    results = conn.search(q, "zhihu", "answer")
    list = []
    for r in results:
        if("title" in r._meta.highlight):
            r['title'] = r._meta.highlight[u"title"][0]
        list.append(r)
    return template('results.html', list=list, count=results.total)

# 需要导入模块: from pyes import ES [as 别名]
# 或者: from pyes.ES import search [as 别名]
class KVStore(KVStoreBase):
    def __init__(self, *args, **kwargs):
        super(KVStore, self).__init__(*args, **kwargs)
        self.connection = ES(settings.THUMBNAIL_ELASTIC_SEARCH_SERVERS)

    def _get_raw(self, key):
        try:
            #import pdb; pdb.set_trace()
            value = self.connection.get(settings.THUMBNAIL_ELASTIC_SEARCH_INDEX, 
                                        settings.THUMBNAIL_ELASTIC_SEARCH_DOCUMENT_TYPE,
                                        key)
            return value['_source']['value']
        except:
            return None

    def _set_raw(self, key, value):
        ret = self.connection.index({"value": value}, 
                                    settings.THUMBNAIL_ELASTIC_SEARCH_INDEX,
                                    settings.THUMBNAIL_ELASTIC_SEARCH_DOCUMENT_TYPE,
                                    key)
        return ret['ok']
    
    def _delete_raw(self, *keys):
        rets = []
        for key in keys:
            try:
                ret = self.connection.delete(settings.THUMBNAIL_ELASTIC_SEARCH_INDEX,
                                             settings.THUMBNAIL_ELASTIC_SEARCH_DOCUMENT_TYPE,
                                             key)
                rets.append(ret['ok'])
            except:
                rets.append(False)
        return rets

    def _find_keys_raw(self, prefix):
        search = Search(query=PrefixQuery("_id", prefix), size=1000, start=0, fields=[])
        results = self.connection.search(search, 
                                         indexes=[settings.THUMBNAIL_ELASTIC_SEARCH_INDEX,], 
                                         doc_types=[settings.THUMBNAIL_ELASTIC_SEARCH_DOCUMENT_TYPE,])
        return [hit['_id'] for hit in results['hits']['hits']]

# 需要导入模块: from pyes import ES [as 别名]
# 或者: from pyes.ES import search [as 别名]
from pyes import ES

es = ES()
index_name = "my_index"
type_name = "my_type"

from utils_pyes import create_and_add_mapping, populate

create_and_add_mapping(es, index_name, type_name)
populate(es, index_name, type_name)

from pyes.query import *
from pyes.aggs import *

q = MatchAllQuery()
q = q.search()
q.get_agg_factory().add(TermsAgg('pterms', field="parsedtext"))

results = es.search(q, indices=index_name, doc_types=type_name)

q = MatchAllQuery()
q = q.search()
q.get_agg_factory().add(DateHistogramAgg('date_add',
    field='date',
    interval='month'))

results = es.search(q, indices=index_name, doc_types=type_name)

es.indices.delete_index(index_name)

# 需要导入模块: from pyes import ES [as 别名]
# 或者: from pyes.ES import search [as 别名]
class EsRestConnection(RestConnection):
    def __init__(self, serverInfo, proto = "http"):
        #serverInfo can be a json object
        #only connect pyes to master es node
        #in the case that other nodes are taken down
        #because http requests will fail
        # TODO: dynamic master node detection
        if isinstance(serverInfo, dict):
            self.ip = serverInfo["ip"]
            self.rest_username = serverInfo["username"]
            self.rest_password = serverInfo["password"]
            self.username = serverInfo["es_username"]
            self.password = serverInfo["es_password"]
            self.port = 9091 #serverInfo["port"]
        else:
            self.ip = serverInfo.ip
            self.rest_username = serverInfo.rest_username
            self.rest_password = serverInfo.rest_password
            self.username = serverInfo.es_username
            self.password = serverInfo.es_password
            self.port = 9091 # serverInfo.port

        self.baseUrl = "http://{0}:{1}/".format(self.ip, self.port)
        self.capiBaseUrl = self.baseUrl
        self.esHttpUrl = "http://{0}:9200".format(self.ip)
        self.http_port = str(int(self.port) + 109)
        self.proto = proto
        self.conn = ES(server=self.esHttpUrl)
        self.manager = managers.Cluster(self.conn)
        self.test_params = TestInputSingleton.input
        self.docs = None

    def get_index_stats(self):
        return ES.index_stats()

    def get_indices(self):
        return self.conn.indices.get_indices()

    def get_indices_as_buckets(self, doc_type='couchbaseDocument'):
        buckets = []
        indices = self.get_indices()

        for index in indices:
            bucket = Bucket()
            q = query.MatchAllQuery()
            docs = self.conn.search(q,index,doc_type)
            bucket.name = index
            bucket.type = "es"
            bucket.port = self.port
            bucket.authType = None
            bucket.saslPassword = self.password
            bucket.nodes = list()

            #vBucketServerMap
            bucketStats = BucketStats()
            bucketStats.itemCount = docs.count()
            bucket.stats = bucketStats
            buckets.append(bucket)
            bucket.master_id = "[email protected]"+self.ip

        return buckets

    def get_bucket(self, bucket_name, doc_type):
        for bucket in self.get_indices_as_buckets(doc_type):
            if bucket.name == bucket_name:
                return bucket
        return

    def get_buckets(self):
        return self.get_indices_as_buckets()

    def delete_index(self, name):
        self.conn.indices.delete_index(name)
        return self.conn.indices.exists_index(name)

    def create_index(self, name):

        if self.conn.indices.exists_index(name):
            self.delete_index(name)

        self.conn.indices.create_index(name)
        return self.conn.indices.exists_index(name)

    def delete_bucket(self, name):
        return self.delete_index(name)

    def create_bucket(self, *args, **kwargs):
        name  = 'default'

        if len(args) > 0:
            name = args[0]
        else:
            name = kwargs['bucket']

        return self.create_index(name)

    def is_ns_server_running(self, timeout_in_seconds=360):
        return True


#.........这里部分代码省略.........

# 需要导入模块: from pyes import ES [as 别名]
# 或者: from pyes.ES import search [as 别名]
from pyes import ES

es = ES()
index_name = "my_index"
type_name = "my_type"

from utils_pyes import create_and_add_mapping, populate

create_and_add_mapping(es, index_name, type_name)
populate(es, index_name, type_name)

from pyes.query import *

q = MatchAllQuery()
q = q.search()
q.facet.add_term_facet('tag')

results = es.search(indices=index_name, doc_types=type_name, query=q)

from pyes.facets import *
q = MatchAllQuery()
q = q.search()
q.facet.facets.append(DateHistogramFacet('date_facet',
    field='date',
    interval='month'))

results = es.search(indices=index_name, doc_types=type_name, query=q)

es.indices.delete_index(index_name)

# 需要导入模块: from pyes import ES [as 别名]
# 或者: from pyes.ES import search [as 别名]
class BaseElasticSearchClient(BaseClient):

    def __init__(self, servers, index=None):
        """
        @param servers: Make sure to include the port with the server address
        @param index: Document index
        @return:
        """
        super(BaseElasticSearchClient, self).__init__()
        self.connection = None
        self.servers = servers

        if index is not None:
            self.index = index if type(index) is list else [index]

    def connect(self, connection_pool=1, bulk_size=10):
        update_connection_pool(connection_pool)

        try:
            self.connection = ES(self.servers, bulk_size=bulk_size)
        except NoServerAvailable:
            self._log.error('Failed to connect to elastic search server')
            return False
        return True

    def close(self):
        self.connection = None

    def _create_term_query(self, must_list):
        # TODO: add remaining conditional list functionality.
        query = BoolQuery()
        for term in must_list:
            query.add_must(term)

    def refresh_index(self, index_name, wait=1):
        self._log.info('ES: Refreshing index {0}'.format(index_name))
        self.connection.indices.refresh(index_name, timesleep=wait)

    def has_index(self, index_name):
        self._log.info('ES: Checking for index {0}'.format(index_name))
        try:
            self.connection.status(index_name)
        except IndexMissingException:
            return False
        return True

    def wait_for_index(self, index_name, wait=30):
        """ Checks to see if an index exists.
        Checks every second for int(X) seconds and returns True if successful
        """
        for i in range(0, int(wait)):
            if self.has_index(index_name):
                return True

            sleep(1)
        return False

    def wait_for_messages(self, name, value, num=1, index=None, max_wait=30):
        """ Wait for a specific number of messages to be returned within a
        specified amount of time.
        Checks every second for {max_wait} seconds and returns a list of msgs
        """
        for i in range(0, int(max_wait)):
            msgs = self.find_term(name=name, value=value, size=1, index=index)
            if len(msgs) == num:
                return msgs
            sleep(1)
        return []

    def delete_index(self, index_name):
        self._log.info('ES: Deleting index {0}'.format(index_name))
        self.connection.delete_index(index_name)

    def find_term(self, name, value, size=10, index=None):
        if not self.connection:
            return

        query = TermQuery(name, value)
        return self.connection.search(query=Search(query, size=size),
                                      indices=index or self.index)

    def find(self, filter_terms, size=10, doc_types=None, index=None):
        if not self.connection:
            return

        query = self._create_term_query(must_list=filter_terms)
        return self.connection.search(query=Search(query, size=size),
                                      indices=index or self.index,
                                      doc_types=doc_types)

    def find_one(self, filter_terms, doc_types=None, index=None):
        if not self.connection:
            return

        results = self.find(filter_terms=filter_terms, size=1,
                            doc_types=doc_types, index=index)
        return results[0] if len(results) > 0 else None

# 需要导入模块: from pyes import ES [as 别名]
# 或者: from pyes.ES import search [as 别名]
class ElasticSearch(object):
    def __init__(self, query):
        self.elastic = ES(settings.SEARCH_HOSTS)
        self.query = QueryParser(query)

    def search(self, index_type='job'):
        if not self.query.is_valid():
            self.results = EmptyResults()
            return
        
        where_filters = [ MatchAllFilter() ]
            
        if self.query.params['where']:
            where_filters= [ QueryFilter(StringQuery(self.query.get_where(), search_fields=['city'],\
                analyze_wildcard=True, default_operator="AND")) ] 

        query = FilteredQuery(
            StringQuery(self.query.params['what'], search_fields=settings.SEARCH_WHAT_FIELDS, default_operator='AND', analyze_wildcard=True)\
                if self.query.params['what'] else MatchAllQuery(),
            ORFilter(self._get_geo_filters(where_filters))
        )

        facets_filter = []
        
        if self.query.params.has_key('company.facet') and len(self.query.params['company.facet']):
            facets_filter.append(TermFilter('company.facet', self.query.params['company.facet']))

        sorting = {'_score': {'order': 'desc' }}
        
        if self.query.params.has_key('sorting'):
            if self.query.params['sorting'] in ['score', 'published_date']:
                sorting = {self.query.params['sorting']: {'order': 'desc' }}

        query = Search(
            query, ORFilter(facets_filter) if len(facets_filter) else [], 
            start=self.query.start_page(), size=settings.PAGINATION_PAGE_SIZE, sort=sorting
        )
        
        query.facet.add_term_facet(
            'company.facet', 
            size=settings.SEARCH_FACETS_SIZE
        )

        self.results = self.elastic.search(query, settings.SEARCH_ALIASES, index_type)
        logger.info('Elastic query: %s\n' % str(query.to_search_json()))

    def get_results(self):
        data = []

        if self.results.total:
            for result in self.results.hits:
                item = {}              
                
                if result.has_key('_source'):
                    item = result['_source']
                    del item['details_url']
                    
                    if item.has_key('title'):
                        item['redirect_url'] = reverse('redirect', kwargs={
                            'slug'   : slugify(result['_source']['title']),
                            'source' : result['_source']['source'],
                            'job_id' : result['_id']
                        })
                    
                    if item.has_key('published_date'):
                        item['published_date_ago'] = timesince(result['_source']['published_date']).encode('utf-8')
    
                    if item.has_key('summary'): item['summary'] = truncatechars(result['_source']['summary'], 350)
                    elif item['content']:       item['summary'] = truncatechars(result['_source']['content'], 350)
                        
                    if item.has_key('image'):
                        item['image'] = '%s/job/thumbs/small/%s' % (settings.MEDIA_URL, item['image'])
                        
                if len(item):
                    data.append(item)
                
        return data
    
    def get_facets(self):
        facets = {}
        
        if self.results.total:
            for facet in self.results.facets:
                if self.results.facets[facet].has_key('terms'):
                    facets[facet] = self.results.facets[facet]['terms']

            if facets.has_key('company.facet'):
                for item in facets['company.facet']:
                    if self.query.params.has_key('company.facet') and len(self.query.params['company.facet'])\
                        and self.query.params['company.facet'] == item['term']:
                            item['url'] = self._get_url({'company.facet': '', 'page': 1})
                            item['active'] = True
                    else:
                        item['url'] = self._get_url({'company.facet': item['term'], 'page': 1})
                                
        return facets

    def list_pages(self):
        if self.results.total <= 0:
            return []
#.........这里部分代码省略.........

# 需要导入模块: from pyes import ES [as 别名]
# 或者: from pyes.ES import search [as 别名]
class ESIndexerBase(object):
    ES_HOST = ES_HOST
    ES_INDEX_NAME = ES_INDEX_NAME
    ES_INDEX_TYPE = 'gene'

    def __init__(self):
        self.conn = ES(self.ES_HOST, default_indexes=[self.ES_INDEX_NAME],
        	           timeout=10.0)
        self.step = 10000

    def create_index(self):
        try:
            print self.conn.open_index(self.ES_INDEX_NAME)
        except IndexMissingException:
            print self.conn.create_index(self.ES_INDEX_NAME)

    def delete_index_type(self, index_type):
        '''Delete all indexes for a given index_type.'''
        index_name = self.ES_INDEX_NAME
#        index_type = self.ES_INDEX_TYPE
        #Check if index_type exists
        mapping = self.conn.get_mapping(index_type, index_name)
        if index_name not in mapping or index_type not in mapping[index_name]:
            print 'Error: index type "%s" does not exist in index "%s".' % (index_type, index_name)
            return
        path = '/%s/%s' % (index_name, index_type)
        if ask('Confirm to delete all data under "%s":' % path) == 'Y':
            return self.conn.delete_mapping(index_name, index_type)

    def index(self, doc, index_type, id=None):
        '''add a doc to the index. If id is not None, the existing doc will be
           updated.
        '''
#        index_type = self.ES_INDEX_TYPE
        return self.conn.index(doc, self.ES_INDEX_NAME, index_type, id=id)

    def delete_index(self, index_type, id):
        '''delete a doc from the index based on passed id.'''
#        index_type = self.ES_INDEX_TYPE
        return self.conn.delete(self.ES_INDEX_NAME, index_type, id)

    def optimize(self):
        return self.conn.optimize(self.ES_INDEX_NAME, wait_for_merge=True)

    def get_field_mapping(self):
        import dataload
        reload(dataload)
        dataload.register_sources()
        return dataload.get_mapping()

    def build_index(self, doc_d, update_mapping=False, bulk=True):
        index_name = self.ES_INDEX_NAME
        index_type = self.ES_INDEX_TYPE

        #Test if index exists
        try:
            print "Opening index...", self.conn.open_index(index_name)
        except NotFoundException:
            print 'Error: index "%s" does not exist. Create it first.' % index_name
            return -1

        try:
            cur_mapping = self.conn.get_mapping(index_type, index_name)
            empty_mapping = False
        except ElasticSearchException:
            #if no existing mapping available for index_type
            #force update_mapping to True
            empty_mapping = True
            update_mapping = True

#        empty_mapping = not cur_mapping[index_name].get(index_type, {})
#        if empty_mapping:
#            #if no existing mapping available for index_type
#            #force update_mapping to True
#            update_mapping = True

        if update_mapping:
            print "Updating mapping...",
            if not empty_mapping:
                print "\n\tRemoving existing mapping...",
                print self.conn.delete_mapping(index_name, index_type)
            _mapping = self.get_field_mapping()
            print self.conn.put_mapping(index_type,
                                   _mapping,
                                   [index_name])
        print "Building index..."
        t0 = time.time()
        for doc_id, doc in doc_d.items():
            self.conn.index(doc, index_name, index_type, doc_id, bulk=bulk)
        print self.conn.flush()
        print self.conn.refresh()
        print "Done[%s]" % timesofar(t0)

    def query(self, qs, fields='symbol,name', **kwargs):
        _q = StringQuery(qs)
        res = self.conn.search(_q, fields=fields, **kwargs)
        return res

# 需要导入模块: from pyes import ES [as 别名]
# 或者: from pyes.ES import search [as 别名]
class ElasticCatalog(object):
    default_indexes = {
        'zelastic_doc_id': {
            'type': 'string',
            'index': 'not_analyzed'
        }
    }

    def __init__(self, connection_string, elastic_name, storage, bulk=False,
                 bulk_size=400):
        self.conn = ES(connection_string, bulk_size=bulk_size)
        self.bulk_size = bulk_size
        self.name = elastic_name
        self.storage = storage
        self.bulk = bulk

    def update_mapping(self, name):
        meta = self.storage.meta(name)
        indexes = meta['indexes']
        properties = self.default_indexes.copy()
        try:
            self.conn.create_index(self.name)
        except IndexAlreadyExistsException:
            pass
        for index_name, _type in indexes.items():
            index = None
            if _type == 'str':
                index = {
                    'type': 'string',
                    'index': 'not_analyzed',
                }
            elif _type == 'full':
                index = {
                    'type': 'string',
                    'index': 'analyzed',
                }
            elif _type == 'bool':
                index = {
                    'type': 'boolean'
                }
            elif _type == 'int':
                index = {
                    'type': 'integer',
                }
            elif _type in ('datetime', 'date'):
                index = {
                    'type': 'date',
                }
            elif _type == 'float':
                index = {
                    'type': 'float',
                }
            if index is not None:
                properties[index_name] = index
        self.conn.indices.put_mapping(
            doc_type=name,
            mapping={
                'ignore_conflicts': True,
                'properties': properties
            },
            indices=[self.name])

    def id(self, container_name, key):
        return '%s-%s' % (container_name, key)

    def index(self, container_name, doc, key):
        # need to add data to the index that isn't actually persisted
        data = {
            'zelastic_doc_id': key
        }
        meta = self.storage.meta(container_name)
        indexes = meta['indexes']
        for index in indexes.keys():
            if index in doc:
                data[index] = doc[index]
        self.conn.index(
            data,
            self.name,
            container_name,
            self.id(container_name, key),
            bulk=self.bulk)

    def delete(self, container_name, key):
        self.conn.delete(
            self.name,
            container_name,
            self.id(container_name, key),
            bulk=self.bulk)

    def delete_all(self, container_name):
        self.conn.delete_mapping(
            self.name,
            container_name)

    def search(self, container_name, query, **kwargs):
        return self.conn.search(
            query,
            indexes=[self.name],
            doc_types=[container_name],
            **kwargs)
#.........这里部分代码省略.........

# 需要导入模块: from pyes import ES [as 别名]
# 或者: from pyes.ES import search [as 别名]
class DocManager():
    """The DocManager class creates a connection to the backend engine and
        adds/removes documents, and in the case of rollback, searches for them.

        The reason for storing id/doc pairs as opposed to doc's is so that
        multiple updates to the same doc reflect the most up to date version as
        opposed to multiple, slightly different versions of a doc.

        We are using elastic native fields for _id and ns, but we also store
        them as fields in the document, due to compatibility issues.
        """

    def __init__(self, url, auto_commit=True, unique_key='_id'):
        """Verify Elastic URL and establish a connection.
        """

        if verify_url(url) is False:
            raise SystemError
        self.elastic = ES(server=url)
        self.auto_commit = auto_commit
        self.doc_type = 'string'  # default type is string, change if needed
        self.unique_key = unique_key
        if auto_commit:
            self.run_auto_commit()

    def stop(self):
        """ Stops the instance
        """
        self.auto_commit = False

    def upsert(self, doc):
        """Update or insert a document into Elastic

        If you'd like to have different types of document in your database,
        you can store the doc type as a field in Mongo and set doc_type to
        that field. (e.g. doc_type = doc['_type'])

        """

        doc_type = self.doc_type
        index = doc['ns']
        doc[self.unique_key] = str(doc[self.unique_key])
        doc_id = doc[self.unique_key]
        id_query = TextQuery('_id', doc_id)
        elastic_cursor = self.elastic.search(query=id_query, indices=index)

        try:
            self.elastic.index(bsjson.dumps(doc), index, doc_type, doc_id)
        except ValueError:
            logging.info("Could not update %s" % (doc,))
        self.elastic.refresh()

    def remove(self, doc):
        """Removes documents from Elastic

        The input is a python dictionary that represents a mongo document.
        """
        try:
            self.elastic.delete(doc['ns'], 'string', str(doc[self.unique_key]))
        except (NotFoundException, TypeMissingException, IndexMissingException):
            pass

    def _remove(self):
        """For test purposes only. Removes all documents in test.test
        """
        try:
            self.elastic.delete('test.test', 'string', '')
        except (NotFoundException, TypeMissingException, IndexMissingException):
            pass

    def search(self, start_ts, end_ts):
        """Called to query Elastic for documents in a time range.
        """
        res = ESRange('_ts', from_value=start_ts, to_value=end_ts)
        results = self.elastic.search(RangeQuery(res))
        return results

    def _search(self):
        """For test purposes only. Performs search on Elastic with empty query.
        Does not have to be implemented.
        """
        results = self.elastic.search(MatchAllQuery())
        return results

    def commit(self):
        """This function is used to force a refresh/commit.
        """
        retry_until_ok(self.elastic.refresh)

    def run_auto_commit(self):
        """Periodically commits to the Elastic server.
        """
        self.elastic.refresh()

        if self.auto_commit:
            Timer(1, self.run_auto_commit).start()

    def get_last_doc(self):
        """Returns the last document stored in the Elastic engine.
        """
#.........这里部分代码省略.........