Python IndexCreator.get_index_client方法代码示例

# 需要导入模块: from socorro.external.es.index_creator import IndexCreator [as 别名]
# 或者: from socorro.external.es.index_creator.IndexCreator import get_index_client [as 别名]
    def test_mapping(self, mapping):
        """Verify that a mapping is correct.

        This function does so by first creating a new, temporary index in
        elasticsearch using the mapping. It then takes some recent crash
        reports that are in elasticsearch and tries to insert them in the
        temporary index. Any failure in any of those steps will raise an
        exception. If any is raised, that means the mapping is incorrect in
        some way (either it doesn't validate against elasticsearch's rules,
        or is not compatible with the data we currently store).

        If no exception is raised, the mapping is likely correct.

        This function is to be used in any place that can change the
        `storage_mapping` field in any Super Search Field.
        Methods `create_field` and `update_field` use it, see above.
        """
        temp_index = 'socorro_mapping_test'

        es_connection = self.get_connection()

        # Import at runtime to avoid dependency circle.
        from socorro.external.es.index_creator import IndexCreator
        index_creator = IndexCreator(self.config)
        try:
            index_creator.create_index(
                temp_index,
                mapping,
            )

            now = datetimeutil.utc_now()
            last_week = now - datetime.timedelta(days=7)
            current_indices = self.generate_list_of_indexes(last_week, now)

            crashes_sample = es_connection.search(
                index=current_indices,
                doc_type=self.config.elasticsearch.elasticsearch_doctype,
                size=self.config.elasticsearch.mapping_test_crash_number,
            )
            crashes = [x['_source'] for x in crashes_sample['hits']['hits']]

            for crash in crashes:
                es_connection.index(
                    index=temp_index,
                    doc_type=self.config.elasticsearch.elasticsearch_doctype,
                    body=crash,
                )
        except elasticsearch.exceptions.ElasticsearchException as e:
            raise BadArgumentError(
                'storage_mapping',
                msg='Indexing existing data in Elasticsearch failed with the '
                    'new mapping. Error is: %s' % str(e),
            )
        finally:
            try:
                index_creator.get_index_client().delete(temp_index)
            except elasticsearch.exceptions.NotFoundError:
                # If the index does not exist (if the index creation failed
                # for example), we don't need to do anything.
                pass

# 需要导入模块: from socorro.external.es.index_creator import IndexCreator [as 别名]
# 或者: from socorro.external.es.index_creator.IndexCreator import get_index_client [as 别名]
    def test_create_emails_index(self):
        index_creator = IndexCreator(config=self.config)
        index_creator.create_emails_index()

        ok_(
            index_creator.get_index_client().exists(
                self.config.elasticsearch.elasticsearch_emails_index
            )
        )

# 需要导入模块: from socorro.external.es.index_creator import IndexCreator [as 别名]
# 或者: from socorro.external.es.index_creator.IndexCreator import get_index_client [as 别名]
class ElasticsearchTestCase(TestCase):
    """Base class for Elastic Search related unit tests. """

    def __init__(self, *args, **kwargs):
        super(ElasticsearchTestCase, self).__init__(*args, **kwargs)

        self.config = self.get_base_config()
        es_context = self.config.elasticsearch.elasticsearch_class(
            config=self.config.elasticsearch
        )

        creator_config = self.get_tuned_config(IndexCreator)

        self.index_creator = IndexCreator(creator_config)
        self.index_client = self.index_creator.get_index_client()

        with es_context() as conn:
            self.connection = conn

    def setUp(self):
        # Create the supersearch fields.
        self.index_super_search_fields()

        self.index_creator.create_socorro_index(
            self.config.elasticsearch.elasticsearch_index
        )

        super(ElasticsearchTestCase, self).setUp()

    def tearDown(self):
        # Clear the test indices.
        self.index_client.delete(
            self.config.elasticsearch.elasticsearch_default_index
        )
        self.index_client.delete(
            self.config.elasticsearch.elasticsearch_index
        )

        super(ElasticsearchTestCase, self).tearDown()

    def get_tuned_config(self, sources, extra_values=None):
        if not isinstance(sources, (list, tuple)):
            sources = [sources]

        mock_logging = mock.Mock()

        config_definitions = []
        for source in sources:
            conf = source.get_required_config()
            conf.add_option('logger', default=mock_logging)
            config_definitions.append(conf)

        values_source = DEFAULT_VALUES.copy()
        values_source.update({'logger': mock_logging})
        if extra_values:
            values_source.update(extra_values)

        config_manager = ConfigurationManager(
            config_definitions,
            app_name='testapp',
            app_version='1.0',
            app_description='Elasticsearch integration tests',
            values_source_list=[environment, values_source],
            argv_source=[],
        )

        return config_manager.get_config()

    def get_base_config(self, es_index=None):
        extra_values = None
        if es_index:
            extra_values = {
                'resource.elasticsearch.elasticsearch_index': es_index
            }

        return self.get_tuned_config(
            ElasticsearchConfig,
            extra_values=extra_values
        )

    def index_super_search_fields(self, fields=None):
        if fields is None:
            fields = SUPERSEARCH_FIELDS

        es_index = self.config.elasticsearch.elasticsearch_default_index

        actions = []
        for name, field in fields.iteritems():
            action = {
                '_index': es_index,
                '_type': 'supersearch_fields',
                '_id': name,
                '_source': field,
            }
            actions.append(action)

        bulk(
            client=self.connection,
            actions=actions,
        )
#.........这里部分代码省略.........

# 需要导入模块: from socorro.external.es.index_creator import IndexCreator [as 别名]
# 或者: from socorro.external.es.index_creator.IndexCreator import get_index_client [as 别名]
class ElasticsearchTestCase(TestCaseWithConfig):
    """Base class for Elastic Search related unit tests"""

    def __init__(self, *args, **kwargs):
        super(ElasticsearchTestCase, self).__init__(*args, **kwargs)

        self.config = self.get_base_config()
        es_context = self.config.elasticsearch.elasticsearch_class(
            config=self.config.elasticsearch
        )

        creator_config = self.get_tuned_config(IndexCreator)

        self.index_creator = IndexCreator(creator_config)
        self.index_client = self.index_creator.get_index_client()

        with es_context() as conn:
            self.connection = conn

    def setUp(self):
        super(ElasticsearchTestCase, self).setUp()
        self.index_creator.create_socorro_index(self.config.elasticsearch.elasticsearch_index)

    def tearDown(self):
        # Clear the test indices.
        self.index_client.delete(
            self.config.elasticsearch.elasticsearch_index
        )

        super(ElasticsearchTestCase, self).tearDown()

    def health_check(self):
        self.connection.cluster.health(
            wait_for_status='yellow',
            request_timeout=5
        )

    def get_url(self):
        """Returns the first url in the elasticsearch_urls list"""
        return self.config.elasticsearch.elasticsearch_urls[0]

    def get_tuned_config(self, sources, extra_values=None):
        values_source = DEFAULT_VALUES.copy()
        if extra_values:
            values_source.update(extra_values)

        return super(ElasticsearchTestCase, self).get_tuned_config(
            sources, values_source
        )

    def get_base_config(self, es_index=None):
        extra_values = None
        if es_index:
            extra_values = {
                'resource.elasticsearch.elasticsearch_index': es_index
            }

        return self.get_tuned_config(
            ElasticsearchConfig,
            extra_values=extra_values
        )

    def index_crash(self, processed_crash=None, raw_crash=None, crash_id=None):
        if crash_id is None:
            crash_id = str(uuid.UUID(int=random.getrandbits(128)))

        raw_crash = raw_crash or {}
        processed_crash = processed_crash or {}

        doc = {
            'crash_id': crash_id,
            'processed_crash': processed_crash,
            'raw_crash': raw_crash,
        }
        res = self.connection.index(
            index=self.config.elasticsearch.elasticsearch_index,
            doc_type=self.config.elasticsearch.elasticsearch_doctype,
            id=crash_id,
            body=doc,
        )
        return res['_id']

    def index_many_crashes(
        self, number, processed_crash=None, raw_crash=None, loop_field=None
    ):
        processed_crash = processed_crash or {}
        raw_crash = raw_crash or {}

        actions = []
        for i in range(number):
            crash_id = str(uuid.UUID(int=random.getrandbits(128)))

            if loop_field is not None:
                processed_copy = processed_crash.copy()
                processed_copy[loop_field] = processed_crash[loop_field] % i
            else:
                processed_copy = processed_crash

            doc = {
                'crash_id': crash_id,
#.........这里部分代码省略.........