Python models.WebappIndexer类代码示例

 def tearDown(self):
     # Cleanup to remove these from the index.
     self.app1.delete()
     self.app2.delete()
     unindex_webapps([self.app1.id, self.app2.id])
     # Required to purge the suggestions data structure. In Lucene, a
     # document is not deleted from a segment, just marked as deleted.
     WebappIndexer.get_es().optimize(WebappIndexer.get_index(), only_expunge_deletes=True)

def index_webapp(ids, **kw):
    index = kw.pop('index', None) or ALIAS
    sys.stdout.write('Indexing %s apps' % len(ids))

    qs = Webapp.indexing_transformer(Webapp.uncached.filter(id__in=ids))

    docs = [WebappIndexer.extract_document(obj.id, obj=obj) for obj in qs]
    WebappIndexer.bulk_index(docs, es=ES, index=index)

def index_webapp(ids, **kw):
    index = kw.pop("index", None) or ALIAS
    sys.stdout.write("Indexing %s apps" % len(ids))

    qs = Webapp.indexing_transformer(Webapp.with_deleted.no_cache().filter(id__in=ids))

    docs = []
    for obj in qs:
        try:
            docs.append(WebappIndexer.extract_document(obj.id, obj=obj))
        except:
            sys.stdout.write("Failed to index obj: {0}".format(obj.id))

    WebappIndexer.bulk_index(docs, es=ES, index=index)

def index_webapp(ids, **kw):
    index = kw.pop('index', None) or ALIAS
    sys.stdout.write('Indexing %s apps' % len(ids))

    qs = Webapp.indexing_transformer(Webapp.uncached.filter(id__in=ids))

    docs = []
    for obj in qs:
        try:
            docs.append(WebappIndexer.extract_document(obj.id, obj=obj))
        except:
            sys.stdout.write('Failed to index obj: {0}'.format(obj.id))

    WebappIndexer.bulk_index(docs, es=ES, index=index)

def index_webapps(ids, **kw):
    task_log.info('Indexing apps %s-%s. [%s]' % (ids[0], ids[-1], len(ids)))

    index = kw.pop('index', WebappIndexer.get_index())
    # Note: If reindexing is currently occurring, `get_indices` will return
    # more than one index.
    indices = get_indices(index)

    es = WebappIndexer.get_es(urls=settings.ES_URLS)
    qs = Webapp.indexing_transformer(Webapp.uncached.filter(id__in=ids))
    for obj in qs:
        doc = WebappIndexer.extract_document(obj.id, obj)
        for idx in indices:
            WebappIndexer.index(doc, id_=obj.id, es=es, index=idx)

def unindex_webapps(ids, **kw):
    task_log.info("Un-indexing apps %s-%s. [%s]" % (ids[0], ids[-1], len(ids)))

    index = kw.pop("index", WebappIndexer.get_index())
    # Note: If reindexing is currently occurring, `get_indices` will return
    # more than one index.
    indices = get_indices(index)

    es = WebappIndexer.get_es(urls=settings.ES_URLS)
    for id_ in ids:
        for idx in indices:
            try:
                WebappIndexer.unindex(id_=id_, es=es, index=idx)
            except ElasticHttpNotFoundError:
                # Ignore if it's not there.
                task_log.info(u"[Webapp:%s] Unindexing app but not found in index" % id_)

 def test_mapping_properties(self):
     # Spot check a few of the key properties.
     mapping = WebappIndexer.get_mapping()
     keys = mapping['webapp']['properties'].keys()
     for k in ('id', 'app_slug', 'category', 'default_locale',
               'description', 'device', 'features', 'name', 'status'):
         ok_(k in keys, 'Key %s not found in mapping properties' % k)

def create_index(new_index, alias, settings):
    """Creates a mapping for the new index.

    - new_index: new index name
    - alias: alias name
    - settings: a dictionary of settings

    """
    sys.stdout.write(
        'Create the mapping for index %r, alias: %r' % (new_index, alias))

    # Update settings with mapping.
    settings = {
        'settings': settings,
        'mappings': WebappIndexer.get_mapping(),
    }

    # Create index and mapping.
    try:
        ES.create_index(new_index, settings)
    except pyelasticsearch.exceptions.IndexAlreadyExistsError:
        raise CommandError('New index [%s] already exists' % new_index)

    # Don't return until the health is green. By default waits for 30s.
    ES.health(new_index, wait_for_status='green', wait_for_relocating_shards=0)

    def handle(self, *args, **kwargs):
        index = WebappIndexer.get_index()
        doctype = WebappIndexer.get_mapping_type_name()
        es = WebappIndexer.get_es()

        apps = Webapp.objects.values_list('id', flat=True)

        missing_ids = []

        for app in apps:
            try:
                res = es.get(index, doctype, app, fields='id')
            except ElasticHttpNotFoundError:
                # App doesn't exist in our index, add it to `missing_ids`.
                missing_ids.append(app)

        if missing_ids:
            sys.stdout.write('Adding %s doc(s) to the index.'
                             % len(missing_ids))
            index_webapps.delay(missing_ids)
        else:
            sys.stdout.write('No docs missing from index.')

def run_indexing(index):
    """Index the objects.

    - index: name of the index

    Note: Our ES doc sizes are about 5k in size. Chunking by 100 sends ~500kb
    of data to ES at a time.

    TODO: Use celery chords here to parallelize these indexing chunks. This
          requires celery 3 (bug 825938).

    """
    sys.stdout.write('Indexing apps into index: %s' % index)

    qs = WebappIndexer.get_indexable()
    for chunk in chunked(list(qs), 100):
        index_webapp(chunk, index=index)

    def get(self, request, *args, **kwargs):
        limit = request.GET.get("limit", 5)
        es_query = {
            "apps": {"completion": {"field": "name_suggest", "size": limit}, "text": request.GET.get("q", "").strip()}
        }

        results = S(WebappIndexer).get_es().send_request("GET", [WebappIndexer.get_index(), "_suggest"], body=es_query)

        if "apps" in results:
            data = results["apps"][0]["options"]
        else:
            data = []
        serializer = self.get_serializer(data)
        # This returns a JSON list. Usually this is a bad idea for security
        # reasons, but we don't include any user-specific data, it's fully
        # anonymous, so we're fine.
        return HttpResponse(json.dumps(serializer.data), content_type="application/x-rocketbar+json")

    def test_q_num_requests_no_results(self):
        es = WebappIndexer.get_es()
        orig_search = es.search
        es.counter = 0

        def monkey_search(*args, **kwargs):
            es.counter += 1
            return orig_search(*args, **kwargs)

        es.search = monkey_search

        res = self.client.get(self.url, data={'q': 'noresults'})
        eq_(res.status_code, 200)
        eq_(res.json['meta']['total_count'], 0)
        eq_(len(res.json['objects']), 0)

        # Verify only one search call was made.
        eq_(es.counter, 1)

        es.search = orig_search

    def test_q_num_requests(self):
        es = WebappIndexer.get_es()
        orig_search = es.search
        es.counter = 0

        def monkey_search(*args, **kwargs):
            es.counter += 1
            return orig_search(*args, **kwargs)

        es.search = monkey_search

        res = self.client.get(self.url, data={'q': 'something'})
        eq_(res.status_code, 200)
        obj = res.json['objects'][0]
        eq_(obj['slug'], self.webapp.app_slug)

        # Verify only one search call was made.
        eq_(es.counter, 1)

        es.search = orig_search

    def test_q_num_requests(self):
        es = WebappIndexer.get_es()
        orig_search = es.search
        es.counter = 0

        def monkey_search(*args, **kwargs):
            es.counter += 1
            return orig_search(*args, **kwargs)

        es.search = monkey_search

        res = self.client.get(self.url, data={"q": "something"})
        eq_(res.status_code, 200)
        eq_(res.json["meta"]["total_count"], 1)
        eq_(len(res.json["objects"]), 1)
        obj = res.json["objects"][0]
        eq_(obj["slug"], self.webapp.app_slug)

        # Verify only one search call was made.
        eq_(es.counter, 1)

        es.search = orig_search

    def get(self, request, *args, **kwargs):
        limit = request.GET.get('limit', 5)
        es_query = {
            'apps': {
                'completion': {'field': 'name_suggest', 'size': limit},
                'text': request.GET.get('q', '').strip()
            }
        }

        results = S(WebappIndexer).get_es().send_request(
            'GET', [WebappIndexer.get_index(), '_suggest'], body=es_query)

        if 'apps' in results:
            data = results['apps'][0]['options']
        else:
            data = []
        serializer = self.get_serializer(data)
        # This returns a JSON list. Usually this is a bad idea for security
        # reasons, but we don't include any user-specific data, it's fully
        # anonymous, so we're fine.
        return HttpResponse(json.dumps(serializer.data),
                            content_type='application/x-rocketbar+json')