Python supersearch.SuperSearch类代码示例

    def test_get(self):
        """Test a search with default values returns the right structure. """
        with _get_config_manager().context() as config:
            api = SuperSearch(config)

        res = api.get()

        self.assertTrue('total' in res)
        self.assertEqual(res['total'], 21)

        self.assertTrue('hits' in res)
        self.assertEqual(len(res['hits']), res['total'])

        self.assertTrue('facets' in res)
        self.assertTrue('signature' in res['facets'])

        expected_signatures = [
            {'term': 'js::break_your_browser', 'count': 20},
            {'term': 'my_bad', 'count': 1},
        ]
        self.assertEqual(res['facets']['signature'], expected_signatures)

        # Test fields are being renamed
        self.assertTrue('date' in res['hits'][0])  # date_processed > date
        self.assertTrue('build_id' in res['hits'][0])  # build > build_id
        self.assertTrue('platform' in res['hits'][0])  # os_name > platform

    def test_get_indexes(self):
        with _get_config_manager(self.config).context() as config:
            api = SuperSearch(config)

        now = datetime.datetime(2000, 2, 1, 0, 0)
        lastweek = now - datetime.timedelta(weeks=1)
        lastmonth = now - datetime.timedelta(weeks=4)

        dates = [search_common.SearchParam("date", now, "<"), search_common.SearchParam("date", lastweek, ">")]

        res = api.get_indexes(dates)
        self.assertEqual(res, "socorro_integration_test")

        with _get_config_manager(self.config, es_index="socorro_%Y%W").context() as config:
            api = SuperSearch(config)

        dates = [search_common.SearchParam("date", now, "<"), search_common.SearchParam("date", lastweek, ">")]

        res = api.get_indexes(dates)
        self.assertEqual(res, "socorro_200004,socorro_200005")

        dates = [search_common.SearchParam("date", now, "<"), search_common.SearchParam("date", lastmonth, ">")]

        res = api.get_indexes(dates)
        self.assertEqual(res, "socorro_200001,socorro_200002,socorro_200003,socorro_200004," "socorro_200005")

    def test_get_with_pagination(self):
        """Test a search with pagination returns expected results. """
        with _get_config_manager().context() as config:
            api = SuperSearch(config)

        args = {
            '_results_number': '10',
        }
        res = api.get(**args)
        self.assertEqual(res['total'], 21)
        self.assertEqual(len(res['hits']), 10)

        args = {
            '_results_number': '10',
            '_results_offset': '10',
        }
        res = api.get(**args)
        self.assertEqual(res['total'], 21)
        self.assertEqual(len(res['hits']), 10)

        args = {
            '_results_number': '10',
            '_results_offset': '15',
        }
        res = api.get(**args)
        self.assertEqual(res['total'], 21)
        self.assertEqual(len(res['hits']), 6)

        args = {
            '_results_number': '10',
            '_results_offset': '30',
        }
        res = api.get(**args)
        self.assertEqual(res['total'], 21)
        self.assertEqual(len(res['hits']), 0)

    def test_get_with_range_operators(self):
        """Test a search with several filters and operators returns expected
        results. """
        with _get_config_manager(self.config).context() as config:
            api = SuperSearch(config)

        # Test date
        now = datetimeutil.utc_now()
        lastweek = now - datetime.timedelta(days=7)
        lastmonth = lastweek - datetime.timedelta(weeks=4)
        args = {"date": ["<%s" % lastweek, ">=%s" % lastmonth]}
        res = api.get(**args)
        self.assertEqual(res["total"], 1)
        self.assertEqual(res["hits"][0]["signature"], "my_little_signature")

        # Test build id
        args = {"build_id": "<1234567890"}
        res = api.get(**args)
        self.assertEqual(res["total"], 1)
        self.assertEqual(res["hits"][0]["signature"], "js::break_your_browser")
        self.assertTrue(res["hits"][0]["build_id"] < 1234567890)

        args = {"build_id": ">1234567889"}
        res = api.get(**args)
        self.assertEqual(res["total"], 20)
        self.assertTrue(res["hits"])
        for report in res["hits"]:
            self.assertTrue(report["build_id"] > 1234567889)

        args = {"build_id": "<=1234567890"}
        res = api.get(**args)
        self.assertEqual(res["total"], 21)
        self.assertTrue(res["hits"])
        for report in res["hits"]:
            self.assertTrue(report["build_id"] <= 1234567890)

    def test_get_with_not_operator(self):
        """Test a search with a few NOT operators. """
        with _get_config_manager(self.config).context() as config:
            api = SuperSearch(config)

        # Test signature
        args = {"signature": ["js", "break_your_browser"]}
        res = api.get(**args)
        self.assertEqual(res["total"], 20)
        self.assertTrue(res["hits"])
        for report in res["hits"]:
            self.assertEqual(report["signature"], "js::break_your_browser")

        # - Test contains mode
        args = {"signature": "!~bad"}
        res = api.get(**args)
        self.assertEqual(res["total"], 20)

        # - Test is_exactly mode
        args = {"signature": "!=js::break_your_browser"}
        res = api.get(**args)
        self.assertEqual(res["total"], 1)
        self.assertEqual(res["hits"][0]["signature"], "my_bad")

        # - Test starts_with mode
        args = {"signature": "!$js"}
        res = api.get(**args)
        self.assertEqual(res["total"], 1)
        self.assertEqual(res["hits"][0]["signature"], "my_bad")

        # - Test ends_with mode
        args = {"signature": "!^browser"}
        res = api.get(**args)
        self.assertEqual(res["total"], 1)
        self.assertEqual(res["hits"][0]["signature"], "my_bad")

        # Test build id
        args = {"build_id": "!<1234567890"}
        res = api.get(**args)
        self.assertEqual(res["total"], 20)
        self.assertTrue(res["hits"])
        for report in res["hits"]:
            self.assertTrue(report["build_id"] > 1234567889)

        args = {"build_id": "!>1234567889"}
        res = api.get(**args)
        self.assertEqual(res["total"], 1)
        self.assertEqual(res["hits"][0]["signature"], "js::break_your_browser")
        self.assertTrue(res["hits"][0]["build_id"] < 1234567890)

        args = {"build_id": "!<=1234567890"}
        res = api.get(**args)
        self.assertEqual(res["total"], 0)

    def setUp(self):
        super(IntegrationTestSettings, self).setUp()

        config = self.get_config_context()
        self.storage = crashstorage.ElasticSearchCrashStorage(config)

        # clear the indices cache so the index is created on every test
        self.storage.indices_cache = set()

        self.now = utc_now()

        # Create the index that will be used.
        es_index = self.storage.get_index_for_crash(self.now)
        self.storage.create_socorro_index(es_index)

        # Create the supersearch fields.
        self.storage.es.bulk_index(
            index=config.webapi.elasticsearch_default_index,
            doc_type='supersearch_fields',
            docs=SUPERSEARCH_FIELDS.values(),
            id_field='name',
            refresh=True,
        )

        self.api = SuperSearch(config=config)

        # This an ugly hack to give elasticsearch some time to finish creating
        # the new index. It is needed for jenkins only, because we have a
        # special case here where we index only one or two documents before
        # querying. Other tests are not affected.
        # TODO: try to remove it, or at least understand why it is needed.
        time.sleep(1)

    def setUp(self):
        super(IntegrationTestSettings, self).setUp()

        config = self.get_config_context()
        self.storage = crashstorage.ElasticSearchCrashStorage(config)

        # clear the indices cache so the index is created on every test
        self.storage.indices_cache = set()

        self.now = utc_now()

        # Create the supersearch fields.
        self.storage.es.bulk_index(
            index=config.webapi.elasticsearch_default_index,
            doc_type='supersearch_fields',
            docs=SUPERSEARCH_FIELDS.values(),
            id_field='name',
            refresh=True,
        )

        # Create the index that will be used.
        es_index = self.storage.get_index_for_crash(self.now)
        self.storage.create_socorro_index(es_index)

        self.api = SuperSearch(config=config)

    def test_get_with_range_operators(self):
        """Test a search with several filters and operators returns expected
        results. """
        with _get_config_manager().context() as config:
            api = SuperSearch(config)

        # Test date
        now = datetimeutil.utc_now()
        lastweek = now - datetime.timedelta(days=7)
        lastmonth = lastweek - datetime.timedelta(weeks=4)
        args = {
            'date': [
                '<%s' % lastweek,
                '>=%s' % lastmonth,
            ]
        }
        res = api.get(**args)
        self.assertEqual(res['total'], 1)
        self.assertEqual(res['hits'][0]['signature'], 'my_little_signature')

        # Test build id
        args = {
            'build_id': '<1234567890',
        }
        res = api.get(**args)
        self.assertEqual(res['total'], 1)
        self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser')
        self.assertTrue(res['hits'][0]['build'] < 1234567890)

        args = {
            'build_id': '>1234567889',
        }
        res = api.get(**args)
        self.assertEqual(res['total'], 20)
        self.assertTrue(res['hits'])
        for report in res['hits']:
            self.assertTrue(report['build'] > 1234567889)

        args = {
            'build_id': '<=1234567890',
        }
        res = api.get(**args)
        self.assertEqual(res['total'], 21)
        self.assertTrue(res['hits'])
        for report in res['hits']:
            self.assertTrue(report['build'] <= 1234567890)

    def test_get_indexes(self):
        config = self.get_config_context()
        api = SuperSearch(config=config)

        now = datetime.datetime(2000, 2, 1, 0, 0)
        lastweek = now - datetime.timedelta(weeks=1)
        lastmonth = now - datetime.timedelta(weeks=4)

        dates = [
            search_common.SearchParam('date', now, '<'),
            search_common.SearchParam('date', lastweek, '>'),
        ]

        res = api.get_indexes(dates)
        eq_(res, ['socorro_integration_test'])

        config = self.get_config_context(es_index='socorro_%Y%W')
        api = SuperSearch(config=config)

        dates = [
            search_common.SearchParam('date', now, '<'),
            search_common.SearchParam('date', lastweek, '>'),
        ]

        res = api.get_indexes(dates)
        eq_(res, ['socorro_200004', 'socorro_200005'])

        dates = [
            search_common.SearchParam('date', now, '<'),
            search_common.SearchParam('date', lastmonth, '>'),
        ]

        res = api.get_indexes(dates)
        eq_(
            res,
            [
                'socorro_200001', 'socorro_200002', 'socorro_200003',
                'socorro_200004', 'socorro_200005'
            ]
        )

    def test_get_with_facets(self):
        """Test a search with facets returns expected results. """
        with _get_config_manager(self.config).context() as config:
            api = SuperSearch(config)

        # Test several facets
        args = {"_facets": ["signature", "platform"]}
        res = api.get(**args)

        self.assertTrue("facets" in res)
        self.assertTrue("signature" in res["facets"])

        expected_signatures = [{"term": "js::break_your_browser", "count": 20}, {"term": "my_bad", "count": 1}]
        self.assertEqual(res["facets"]["signature"], expected_signatures)

        self.assertTrue("platform" in res["facets"])
        expected_platforms = [{"term": "Linux", "count": 20}, {"term": "Windows NT", "count": 1}]
        self.assertEqual(res["facets"]["platform"], expected_platforms)

        # Test one facet with filters
        args = {"_facets": ["release_channel"], "release_channel": "aurora"}
        res = api.get(**args)

        self.assertTrue("release_channel" in res["facets"])

        expected_signatures = [{"term": "aurora", "count": 1}]
        self.assertEqual(res["facets"]["release_channel"], expected_signatures)

        # Test one facet with a different filter
        args = {"_facets": ["release_channel"], "platform": "linux"}
        res = api.get(**args)

        self.assertTrue("release_channel" in res["facets"])

        expected_signatures = [{"term": "release", "count": 19}, {"term": "aurora", "count": 1}]
        self.assertEqual(res["facets"]["release_channel"], expected_signatures)

        # Test errors
        self.assertRaises(BadArgumentError, api.get, _facets=["unkownfield"])

    def test_get_indexes(self):
        with _get_config_manager(self.config).context() as config:
            api = SuperSearch(config)

        now = datetime.datetime(2000, 2, 1, 0, 0)
        lastweek = now - datetime.timedelta(weeks=1)
        lastmonth = now - datetime.timedelta(weeks=4)

        dates = [
            search_common.SearchParam('date', now, '<'),
            search_common.SearchParam('date', lastweek, '>'),
        ]

        res = api.get_indexes(dates)
        self.assertEqual(res, 'socorro_integration_test')

        with _get_config_manager(self.config, es_index='socorro_%Y%W') \
            .context() as config:
            api = SuperSearch(config)

        dates = [
            search_common.SearchParam('date', now, '<'),
            search_common.SearchParam('date', lastweek, '>'),
        ]

        res = api.get_indexes(dates)
        self.assertEqual(res, 'socorro_200004,socorro_200005')

        dates = [
            search_common.SearchParam('date', now, '<'),
            search_common.SearchParam('date', lastmonth, '>'),
        ]

        res = api.get_indexes(dates)
        self.assertEqual(
            res,
            'socorro_200001,socorro_200002,socorro_200003,socorro_200004,'
            'socorro_200005'
        )

    def test_get(self):
        """Test a search with default values returns the right structure. """
        with _get_config_manager(self.config).context() as config:
            api = SuperSearch(config)

        res = api.get()

        self.assertTrue("total" in res)
        self.assertEqual(res["total"], 21)

        self.assertTrue("hits" in res)
        self.assertEqual(len(res["hits"]), res["total"])

        self.assertTrue("facets" in res)
        self.assertTrue("signature" in res["facets"])

        expected_signatures = [{"term": "js::break_your_browser", "count": 20}, {"term": "my_bad", "count": 1}]
        self.assertEqual(res["facets"]["signature"], expected_signatures)

        # Test fields are being renamed
        self.assertTrue("date" in res["hits"][0])  # date_processed > date
        self.assertTrue("build_id" in res["hits"][0])  # build > build_id
        self.assertTrue("platform" in res["hits"][0])  # os_name > platform

    def test_get_with_pagination(self):
        """Test a search with pagination returns expected results. """
        with _get_config_manager(self.config).context() as config:
            api = SuperSearch(config)

        args = {"_results_number": "10"}
        res = api.get(**args)
        self.assertEqual(res["total"], 21)
        self.assertEqual(len(res["hits"]), 10)

        args = {"_results_number": "10", "_results_offset": "10"}
        res = api.get(**args)
        self.assertEqual(res["total"], 21)
        self.assertEqual(len(res["hits"]), 10)

        args = {"_results_number": "10", "_results_offset": "15"}
        res = api.get(**args)
        self.assertEqual(res["total"], 21)
        self.assertEqual(len(res["hits"]), 6)

        args = {"_results_number": "10", "_results_offset": "30"}
        res = api.get(**args)
        self.assertEqual(res["total"], 21)
        self.assertEqual(len(res["hits"]), 0)

    def setUp(self):
        super(IntegrationTestSettings, self).setUp()

        config = self.get_config_context()
        self.storage = crashstorage.ElasticSearchCrashStorage(config)
        self.api = SuperSearch(config)

        # clear the indices cache so the index is created on every test
        self.storage.indices_cache = set()

        self.now = utc_now()

        # Create the index that will be used.
        es_index = self.storage.get_index_for_crash(self.now)
        self.storage.create_socorro_index(es_index)

        # This an ugly hack to give elasticsearch some time to finish creating
        # the new index. It is needed for jenkins only, because we have a
        # special case here where we index only one or two documents before
        # querying. Other tests are not affected.
        # TODO: try to remove it, or at least understand why it is needed.
        time.sleep(1)

class IntegrationTestSettings(ElasticSearchTestCase):
    """Test the settings and mappings used in elasticsearch, through
    the supersearch service. """

    def setUp(self):
        super(IntegrationTestSettings, self).setUp()

        config = self.get_config_context()
        self.storage = crashstorage.ElasticSearchCrashStorage(config)
        self.api = SuperSearch(config=config)

        # clear the indices cache so the index is created on every test
        self.storage.indices_cache = set()

        self.now = utc_now()

        # Create the index that will be used.
        es_index = self.storage.get_index_for_crash(self.now)
        self.storage.create_socorro_index(es_index)

        # This an ugly hack to give elasticsearch some time to finish creating
        # the new index. It is needed for jenkins only, because we have a
        # special case here where we index only one or two documents before
        # querying. Other tests are not affected.
        # TODO: try to remove it, or at least understand why it is needed.
        time.sleep(1)

    def tearDown(self):
        # clear the test index
        config = self.get_config_context()
        self.storage.es.delete_index(config.webapi.elasticsearch_index)

        super(IntegrationTestSettings, self).tearDown()

    def test_dump_field(self):
        """Verify that the 'dump' field can be queried as expected. """
        # Index some data.
        processed_crash = {
            'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100',
            'date_processed': self.now,
            'dump': EXAMPLE_DUMP,
        }
        self.storage.save_processed(processed_crash)
        self.storage.es.refresh()

        # Simple test with one word, no upper case.
        res = self.api.get(dump='~family')
        self.assertEqual(res['total'], 1)

        # Several words, with upper case.
        res = self.api.get(dump='~Windows NT')
        self.assertEqual(res['total'], 1)

    def test_cpu_info_field(self):
        """Verify that the 'cpu_info' field can be queried as expected. """
        processed_crash = {
            'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101',
            'date_processed': self.now,
            'cpu_info': 'GenuineIntel family 6 model 15 stepping 13',
        }
        self.storage.save_processed(processed_crash)
        self.storage.es.refresh()

        # Simple test with one word, no upper case.
        res = self.api.get(cpu_info='~model')
        self.assertEqual(res['total'], 1)
        self.assertTrue('model' in res['hits'][0]['cpu_info'])

        # Several words, with upper case, 'starts with' mode.
        res = self.api.get(cpu_info='$GenuineIntel family')
        self.assertEqual(res['total'], 1)
        self.assertTrue('GenuineIntel family' in res['hits'][0]['cpu_info'])

    def test_dom_ipc_enabled_field(self):
        """Verify that the 'dom_ipc_enabled' field can be queried as
        expected. """
        processed_crash = {
            'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101',
            'date_processed': self.now,
        }
        raw_crash = {
            'DOMIPCEnabled': True,
        }
        self.storage.save_raw_and_processed(
            raw_crash, None, processed_crash, processed_crash['uuid']
        )
        processed_crash = {
            'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120102',
            'date_processed': self.now,
        }
        raw_crash = {
            'DOMIPCEnabled': False,
        }
        self.storage.save_raw_and_processed(
            raw_crash, None, processed_crash, processed_crash['uuid']
        )
        processed_crash = {
            'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120103',
            'date_processed': self.now,
        }
#.........这里部分代码省略.........