Python elasticsearch.ES类代码示例

    def index_models(self, model_names):
        self.log.info('Indexing models documents')
        params = self.options.params or ''
        params = dict([[k, v[0]] for k, v in urllib.parse.parse_qs(params).items()])

        for model_name in model_names:
            self.log.info('Processing model `{}`'.format(model_name))
            model = engine.get_document_cls(model_name)

            local_params = dict()
            local_params.update(params)

            if '_limit' not in local_params:
                limit = model.get_collection().count()
                local_params['_limit'] = limit

            chunk_size = int(self.options.chunk or local_params['_limit'])

            es = ES(source=model_name, index_name=self.options.index,
                    chunk_size=chunk_size)

            query_set = model.get_collection(**local_params)
            documents = to_indexable_dicts(query_set)
            self.log.info('Indexing missing `{}` documents'.format(
                model_name))
            es.index_missing_documents(documents)

def includeme(config):
    Settings = dictset(config.registry.settings)
    config.include("nefertari.engine")
    config.include("nefertari")
    config.include("nefertari.view")
    config.include("nefertari.elasticsearch")

    # Process nefertari settings
    if Settings.asbool("debug"):
        log.warning("*** DEBUG DEBUG DEBUG mode ***")
        config.add_tween("nefertari.tweens.get_tunneling")

    if Settings.asbool("cors.enable"):
        config.add_tween("nefertari.tweens.cors")

    if Settings.asbool("ssl_middleware.enable"):
        config.add_tween("nefertari.tweens.ssl")

    if Settings.asbool("request_timing.enable"):
        config.add_tween("nefertari.tweens.request_timing")

    # Set root factory
    config.root_factory = NefertariRootACL

    # Process auth settings
    root = config.get_root_resource()
    ramses_auth = Settings.asbool("ramses.auth", False)
    root.auth = ramses_auth

    log.info("Parsing RAML")
    parsed_raml = pyraml.parser.load(Settings["ramses.raml_schema"])

    log.info("Starting models generation")
    generate_models(config, raml_resources=parsed_raml.resources)

    if ramses_auth:
        if getattr(config.registry, "auth_model", None) is None:
            from nefertari.authentication.models import AuthUser

            config.registry.auth_model = AuthUser
        from .auth import setup_auth_policies

        setup_auth_policies(config, parsed_raml)

    log.info("Starting server generation")
    generate_server(parsed_raml, config)

    log.info("Running nefertari.engine.setup_database")
    from nefertari.engine import setup_database

    setup_database(config)

    from nefertari.elasticsearch import ES

    ES.setup_mappings()

    if ramses_auth:
        config.include("ramses.auth")

    log.info("Server succesfully generated\n")

 def recreate_index(self):
     self.log.info('Deleting index')
     ES.delete_index()
     self.log.info('Creating index')
     create_index_with_settings(self.settings)
     self.log.info('Creating mappings')
     ES.setup_mappings()

def on_post_bulk_insert(sender,documents,**kw):
    if not documents:
        return
    from nefertari.elasticsearch import ES
    es = ES(source=documents[0].__class__.__name__)
    docs = to_dicts(documents)
    es.index(docs)

 def getitem_es(self, key):
     es = ES(self.item_model.__name__)
     obj = es.get_item(id=key)
     obj.__acl__ = self.item_acl(obj)
     obj.__parent__ = self
     obj.__name__ = key
     return obj

def on_after_delete(mapper, connection, target):
    from nefertari.elasticsearch import ES
    model_cls = target.__class__
    es = ES(model_cls.__name__)
    obj_id = getattr(target, model_cls.pk_field())
    es.delete(obj_id)
    es.index_refs(target)

def on_after_delete(mapper, connection, target):
    from nefertari.elasticsearch import ES
    request = getattr(target, '_request', None)
    model_cls = target.__class__
    es = ES(model_cls.__name__)
    obj_id = getattr(target, model_cls.pk_field())
    es.delete(obj_id, request=request)
    es.index_relations(target, request=request)

def includeme(config):
    Settings = dictset(config.registry.settings)
    config.include('nefertari.engine')
    config.include('nefertari')
    config.include('nefertari.view')

    # Process nefertari settings
    if Settings.asbool('debug'):
        log.warning('*** DEBUG DEBUG DEBUG mode ***')
        config.add_tween('nefertari.tweens.get_tunneling')

    if Settings.asbool('cors.enable'):
        config.add_tween('nefertari.tweens.cors')

    if Settings.asbool('ssl_middleware.enable'):
        config.add_tween('nefertari.tweens.ssl')

    if Settings.asbool('request_timing.enable'):
        config.add_tween('nefertari.tweens.request_timing')

    # Set root factory
    config.root_factory = NefertariRootACL

    # Process auth settings
    root = config.get_root_resource()
    ramses_auth = Settings.asbool('ramses.auth', False)
    root.auth = ramses_auth

    log.info('Parsing RAML')
    parsed_raml = pyraml.parser.load(Settings['ramses.raml_schema'])

    log.info('Starting models generation')
    generate_models(config, raml_resources=parsed_raml.resources)

    if ramses_auth:
        if getattr(config.registry, 'auth_model', None) is None:
            from nefertari.authentication.models import get_authuser_model
            config.registry.auth_model = get_authuser_model()
        from .auth import setup_auth_policies
        setup_auth_policies(config, parsed_raml)

    config.include('nefertari.elasticsearch')

    log.info('Starting server generation')
    generate_server(parsed_raml, config)

    log.info('Running nefertari.engine.setup_database')
    from nefertari.engine import setup_database
    setup_database(config)

    from nefertari.elasticsearch import ES
    ES.setup_mappings()

    if ramses_auth:
        config.include('ramses.auth')

    log.info('Server succesfully generated\n')

 def run(self):
     ES.setup(self.settings)
     if self.options.recreate:
         self.recreate_index()
         models = engine.get_document_classes()
         model_names = [
             name for name, model in models.items()
             if getattr(model, '_index_enabled', False)]
     else:
         model_names = split_strip(self.options.models)
     self.index_models(model_names)

def on_bulk_delete(model_cls, objects, request):
    if not getattr(model_cls, '_index_enabled', False):
        return

    pk_field = model_cls.pk_field()
    ids = [getattr(obj, pk_field) for obj in objects]

    from nefertari.elasticsearch import ES
    es = ES(source=model_cls.__name__)
    es.delete(ids, request=request)

    # Reindex relationships
    es.bulk_index_relations(objects, request=request)

def on_bulk_update(model_cls, objects, request):
    if not getattr(model_cls, '_index_enabled', False):
        return

    if not objects:
        return

    from nefertari.elasticsearch import ES
    es = ES(source=model_cls.__name__)
    documents = to_dicts(objects)
    es.index(documents, request=request)

    # Reindex relationships
    es.bulk_index_relations(objects, request=request, nested_only=True)

 def getitem_es(self, key):
     """ Get item with ID of :key: from elasticsearch """
     from nefertari.elasticsearch import ES
     es = ES(self.__context_class__.__name__)
     pk_field = self.__context_class__.pk_field()
     kwargs = {
         pk_field: key,
         '_limit': 1,
         '__raise_on_empty': True,
     }
     obj = es.get_collection(**kwargs)[0]
     obj.__acl__ = self.context_acl(obj)
     obj.__parent__ = self
     obj.__name__ = key
     return obj

    def get_es_mapping(cls, _depth=None, types_map=None):
        """ Generate ES mapping from model schema. """
        from nefertari.elasticsearch import ES

        if types_map is None:
            types_map = TYPES_MAP

        if _depth is None:
            _depth = cls._nesting_depth

        depth_reached = _depth <= 0

        nested_substitutions = []
        properties = {}
        mapping = {
            ES.src2type(cls.__name__): {
                'properties': properties
            }
        }
        mapper = class_mapper(cls)
        columns = {c.name: c for c in mapper.columns}
        relationships = {r.key: r for r in mapper.relationships}

        for name, column in columns.items():
            column_type = column.type
            if isinstance(column_type, types.ChoiceArray):
                column_type = column_type.impl.item_type
            column_type = type(column_type)
            if column_type not in types_map:
                continue
            properties[name] = types_map[column_type]

            if hasattr(column, "_es_multi_field") and getattr(column, "_es_multi_field"):
                multi_fields = getattr(column, "_es_multi_field")
                properties[name] = properties[name].copy()
                properties[name]["fields"] = {}

                for multi_field_name in multi_fields:
                    properties[name]["fields"][multi_field_name] = multi_fields[multi_field_name].copy()
                    properties[name]["fields"][multi_field_name].update(types_map[column_type])

        for name, column in relationships.items():
            if name in cls._nested_relationships and not depth_reached:
                column_type = {'type': 'nested', 'include_in_parent': True}
                nested_substitutions.append(name)

                submapping, sub_substitutions = column.mapper.class_.get_es_mapping(
                    _depth=_depth - 1)

                column_type.update(list(submapping.values())[0])
                properties[name + "_nested"] = column_type

            rel_pk_field = column.mapper.class_.pk_field_type()
            column_type = types_map[rel_pk_field]
            properties[name] = column_type

        properties['_pk'] = {'type': 'string'}

        return mapping, nested_substitutions

def on_bulk_update(update_context):
    request = getattr(
        update_context.query, '_request', None)
    model_cls = update_context.mapper.entity
    if not getattr(model_cls, '_index_enabled', False):
        return

    objects = update_context.query.all()
    if not objects:
        return

    from nefertari.elasticsearch import ES
    es = ES(source=model_cls.__name__)
    es.index(objects, request=request)

    # Reindex relationships
    es.bulk_index_relations(objects, request=request, nested_only=True)

    def run(self, quiet=False):
        from nefertari.elasticsearch import ES
        ES.setup(self.settings)
        models_paths = split_strip(self.options.models)

        for path in models_paths:
            model = resolve(path)
            model_name = path.split('.')[-1]

            params = self.options.params or ''
            params = dict([
                [k, v[0]] for k, v in urlparse.parse_qs(params).items()
            ])
            params.setdefault('_limit', params.get('_limit', 10000))
            chunk_size = self.options.chunk or params['_limit']

            es = ES(source=model_name, index_name=self.options.index)
            query_set = model.get_collection(**params)
            documents = to_dicts(query_set)

            if self.options.force:
                es.index(documents, chunk_size=chunk_size)
            else:
                es.index_missing(documents, chunk_size=chunk_size)

        return 0