Python SearchQuery.add_aggregation方法代码示例

# 需要导入模块: from mozdef_util.query_models import SearchQuery [as 别名]
# 或者: from mozdef_util.query_models.SearchQuery import add_aggregation [as 别名]
    def test_simple_aggregation_note_field(self):
        events = [
            {"test": "value", "note": "abvc"},
            {"test": "value", "note": "abvc"},
            {"test": "value", "note": "think"},
            {"test": "value", "summary": "think"},
            {"test": "value", "note": "abvc space line"},
        ]
        for event in events:
            self.populate_test_object(event)
        self.refresh(self.event_index_name)

        search_query = SearchQuery()
        search_query.add_must(TermMatch('test', 'value'))
        search_query.add_aggregation(Aggregation('note'))
        results = search_query.execute(self.es_client)

        assert results['aggregations'].keys() == ['note']

        assert results['aggregations']['note'].keys() == ['terms']
        assert len(results['aggregations']['note']['terms']) == 3
        assert results['aggregations']['note']['terms'][0].keys() == ['count', 'key']

        assert results['aggregations']['note']['terms'][0]['count'] == 2
        assert results['aggregations']['note']['terms'][0]['key'] == 'abvc'

        assert results['aggregations']['note']['terms'][1]['count'] == 1
        assert results['aggregations']['note']['terms'][1]['key'] == 'abvc space line'

        assert results['aggregations']['note']['terms'][2]['count'] == 1
        assert results['aggregations']['note']['terms'][2]['key'] == 'think'

# 需要导入模块: from mozdef_util.query_models import SearchQuery [as 别名]
# 或者: from mozdef_util.query_models.SearchQuery import add_aggregation [as 别名]
    def test_aggregation_multiple_layers(self):
        events = [
            {
                "test": "value",
                "details": {"ip": "127.0.0.1"},
            },
            {
                "test": "value",
                "details": {"ip": "127.0.0.1"},
            },
            {
                "test": "value",
                "details": {"ip": "192.168.1.1"},
            },
        ]

        for event in events:
            self.populate_test_object(event)
        self.refresh(self.event_index_name)

        search_query = SearchQuery()
        search_query.add_must(TermMatch('test', 'value'))
        search_query.add_aggregation(Aggregation('details.ip'))
        results = search_query.execute(self.es_client)

        assert results['aggregations'].keys() == ['details.ip']
        assert results['aggregations']['details.ip'].keys() == ['terms']
        assert len(results['aggregations']['details.ip']['terms']) == 2

        assert results['aggregations']['details.ip']['terms'][0]['count'] == 2
        assert results['aggregations']['details.ip']['terms'][0]['key'] == "127.0.0.1"

        assert results['aggregations']['details.ip']['terms'][1]['count'] == 1
        assert results['aggregations']['details.ip']['terms'][1]['key'] == "192.168.1.1"

# 需要导入模块: from mozdef_util.query_models import SearchQuery [as 别名]
# 或者: from mozdef_util.query_models.SearchQuery import add_aggregation [as 别名]
 def get_num_events(self):
     self.refresh('events')
     search_query = SearchQuery()
     search_query.add_must(TermMatch('_type', 'event'))
     search_query.add_aggregation(Aggregation('_type'))
     results = search_query.execute(self.es_client)
     if len(results['aggregations']['_type']['terms']) != 0:
         return results['aggregations']['_type']['terms'][0]['count']
     else:
         return 0

# 需要导入模块: from mozdef_util.query_models import SearchQuery [as 别名]
# 或者: from mozdef_util.query_models.SearchQuery import add_aggregation [as 别名]
    def test_aggregation_with_aggregation_size(self):
        for num in range(0, 100):
            event = {'keyname': 'value' + str(num)}
            self.populate_test_object(event)
        self.refresh(self.event_index_name)

        search_query = SearchQuery()
        search_query.add_must(ExistsMatch('keyname'))
        search_query.add_aggregation(Aggregation('keyname', 2))
        results = search_query.execute(self.es_client)
        assert len(results['aggregations']['keyname']['terms']) == 2

# 需要导入模块: from mozdef_util.query_models import SearchQuery [as 别名]
# 或者: from mozdef_util.query_models.SearchQuery import add_aggregation [as 别名]
    def test_aggregation_without_must_fields(self):
        event = self.generate_default_event()
        event['_source']['utctimestamp'] = event['_source']['utctimestamp']()
        event['_source']['receivedtimestamp'] = event['_source']['receivedtimestamp']()
        self.populate_test_event(event)
        self.refresh(self.event_index_name)

        search_query = SearchQuery(minutes=10)

        search_query.add_aggregation(Aggregation('source'))
        results = search_query.execute(self.es_client)
        assert results['aggregations']['source']['terms'][0]['count'] == 1

# 需要导入模块: from mozdef_util.query_models import SearchQuery [as 别名]
# 或者: from mozdef_util.query_models.SearchQuery import add_aggregation [as 别名]
    def test_aggregation_query_execute(self):
        query = SearchQuery()
        query.add_must(ExistsMatch('note'))
        query.add_aggregation(Aggregation('note'))
        assert query.date_timedelta == {}

        self.populate_example_event()
        self.populate_example_event()
        self.refresh(self.event_index_name)

        results = query.execute(self.es_client)
        assert results.keys() == ['hits', 'meta', 'aggregations']
        assert results['meta'].keys() == ['timed_out']
        assert results['meta']['timed_out'] is False

        assert len(results['hits']) == 2

        assert results['hits'][0].keys() == ['_score', '_type', '_id', '_source', '_index']
        assert type(results['hits'][0]['_id']) == unicode
        assert results['hits'][0]['_type'] == TMP_DOC_TYPE

        assert results['hits'][0]['_index'] == datetime.now().strftime("events-%Y%m%d")

        assert results['hits'][0]['_source']['note'] == 'Example note'
        assert results['hits'][0]['_source']['summary'] == 'Test Summary'
        assert results['hits'][0]['_source']['type'] == 'event'

        assert results['hits'][0]['_source']['details'].keys() == ['information']
        assert results['hits'][0]['_source']['details']['information'] == 'Example information'

        assert results['hits'][1].keys() == ['_score', '_type', '_id', '_source', '_index']
        assert type(results['hits'][1]['_id']) == unicode
        assert results['hits'][1]['_type'] == TMP_DOC_TYPE

        assert results['hits'][1]['_index'] == datetime.now().strftime("events-%Y%m%d")

        assert results['hits'][1]['_source']['note'] == 'Example note'
        assert results['hits'][1]['_source']['summary'] == 'Test Summary'
        assert results['hits'][1]['_source']['type'] == 'event'

        assert results['hits'][1]['_source']['details'].keys() == ['information']
        assert results['hits'][1]['_source']['details']['information'] == 'Example information'

        assert results['aggregations'].keys() == ['note']

        assert results['aggregations']['note'].keys() == ['terms']

        assert len(results['aggregations']['note']['terms']) == 1

        results['aggregations']['note']['terms'].sort()
        assert results['aggregations']['note']['terms'][0]['count'] == 2
        assert results['aggregations']['note']['terms'][0]['key'] == 'Example note'

# 需要导入模块: from mozdef_util.query_models import SearchQuery [as 别名]
# 或者: from mozdef_util.query_models.SearchQuery import add_aggregation [as 别名]
    def test_aggregation_non_existing_layers_term(self):
        events = [
            {"test": "value", "note": "abvc"},
            {"test": "value", "note": "abvc"},
            {"test": "value", "note": "think"},
            {"test": "value", "summary": "think"},
        ]
        for event in events:
            self.populate_test_object(event)
        self.refresh(self.event_index_name)

        search_query = SearchQuery()
        search_query.add_must(TermMatch('test', 'value'))
        search_query.add_aggregation(Aggregation('details.ipinformation'))
        results = search_query.execute(self.es_client)

        assert results['aggregations'].keys() == ['details.ipinformation']
        assert results['aggregations']['details.ipinformation'].keys() == ['terms']
        assert len(results['aggregations']['details.ipinformation']['terms']) == 0

# 需要导入模块: from mozdef_util.query_models import SearchQuery [as 别名]
# 或者: from mozdef_util.query_models.SearchQuery import add_aggregation [as 别名]
def esSearch(es):
    search_query = SearchQuery(minutes=options.aggregationminutes)
    search_query.add_aggregation(Aggregation('category'))
    results = search_query.execute(es)

    mozdefstats = dict(utctimestamp=toUTC(datetime.now()).isoformat())
    mozdefstats['category'] = 'stats'
    mozdefstats['hostname'] = socket.gethostname()
    mozdefstats['mozdefhostname'] = mozdefstats['hostname']
    mozdefstats['severity'] = 'INFO'
    mozdefstats['source'] = 'mozdef'
    mozdefstats['tags'] = ['mozdef', 'stats']
    mozdefstats['summary'] = 'Aggregated category counts'
    mozdefstats['processid'] = os.getpid()
    mozdefstats['processname'] = sys.argv[0]
    mozdefstats['details'] = dict(counts=list())
    for bucket in results['aggregations']['category']['terms']:
        entry = dict()
        entry[bucket['key']] = bucket['count']
        mozdefstats['details']['counts'].append(entry)
    return mozdefstats

# 需要导入模块: from mozdef_util.query_models import SearchQuery [as 别名]
# 或者: from mozdef_util.query_models.SearchQuery import add_aggregation [as 别名]
def verify_events(options):
    es_client = ElasticsearchClient(options.esservers)
    for required_field in options.required_fields:
        logger.debug('Looking for events without ' + required_field)
        search_query = SearchQuery(hours=12)
        search_query.add_must_not(ExistsMatch(required_field))

        # Exclude all events that are mozdef related health and stats
        search_query.add_must_not(TermMatch('_type', 'mozdefstats'))
        search_query.add_must_not(TermMatch('_type', 'mozdefhealth'))

        search_query.add_aggregation(Aggregation('_type'))
        # We don't care about the actual events, we only want the numbers
        results = search_query.execute(es_client, size=1)
        for aggreg_term in results['aggregations']['_type']['terms']:
            count = aggreg_term['count']
            category = aggreg_term['key']
            logger.error("Found {0} bad events of _type '{1}' missing '{2}' field".format(
                count,
                category,
                required_field
            ))

# 需要导入模块: from mozdef_util.query_models import SearchQuery [as 别名]
# 或者: from mozdef_util.query_models.SearchQuery import add_aggregation [as 别名]
    def onMessage(self, request, response):
        '''
        request: http://bottlepy.org/docs/dev/api.html#the-request-object
        response: http://bottlepy.org/docs/dev/api.html#the-response-object

        '''
        # an ES query/facet to count success/failed logins
        # oriented to the data having
        # category: authentication
        # details.success marked true/false for success/failed auth
        # details.username as the user

        begindateUTC=None
        enddateUTC=None
        resultsList = list()
        if begindateUTC is None:
            begindateUTC = datetime.now() - timedelta(hours=12)
            begindateUTC = toUTC(begindateUTC)
        if enddateUTC is None:
            enddateUTC = datetime.now()
            enddateUTC = toUTC(enddateUTC)

        es_client = ElasticsearchClient(list('{0}'.format(s) for s in self.restoptions['esservers']))
        search_query = SearchQuery()
        # a query to tally users with failed logins
        date_range_match = RangeMatch('utctimestamp', begindateUTC, enddateUTC)
        search_query.add_must(date_range_match)
        search_query.add_must(PhraseMatch('category', 'authentication'))
        search_query.add_must(PhraseMatch('details.success','false'))
        search_query.add_must(ExistsMatch('details.username'))
        search_query.add_aggregation(Aggregation('details.success'))
        search_query.add_aggregation(Aggregation('details.username'))

        results = search_query.execute(es_client, indices=['events','events-previous'])

        # any usernames or words to ignore
        # especially useful if ES is analyzing the username field and breaking apart [email protected]
        # into user somewhere and .com
        stoplist =self.options.ignoreusernames.split(',')
        # walk the aggregate failed users
        # and look for successes/failures
        for t in results['aggregations']['details.username']['terms']:
            if t['key'] in stoplist:
                continue
            failures = 0
            success = 0
            username = t['key']

            details_query = SearchQuery()
            details_query.add_must(date_range_match)
            details_query.add_must(PhraseMatch('category', 'authentication'))
            details_query.add_must(PhraseMatch('details.username', username))
            details_query.add_aggregation(Aggregation('details.success'))

            details_results = details_query.execute(es_client)
            # details.success is boolean. As an aggregate is an int (0/1)
            for details_term in details_results['aggregations']['details.success']['terms']:
                if details_term['key'] == 1:
                    success = details_term['count']
                if details_term['key'] == 0:
                    failures = details_term['count']
            resultsList.append(
                dict(
                    username=username,
                    failures=failures,
                    success=success,
                    begin=begindateUTC.isoformat(),
                    end=enddateUTC.isoformat()
                )
            )

        response.body = json.dumps(resultsList)
        response.status = 200

        return (request, response)

# 需要导入模块: from mozdef_util.query_models import SearchQuery [as 别名]
# 或者: from mozdef_util.query_models.SearchQuery import add_aggregation [as 别名]
    def test_complex_aggregation_query_execute(self):
        query = SearchQuery()
        assert query.date_timedelta == {}
        query.add_must(ExistsMatch('ip'))
        query.add_aggregation(Aggregation('ip'))
        self.populate_test_event(
            {
                'summary': 'Test Summary',
                'ip': '127.0.0.1',
                'details': {
                    'information': 'Example information'
                }
            }
        )
        self.populate_test_event(
            {
                'summary': 'Test Summary',
                'ip': '1.2.3.4',
                'details': {
                    'information': 'Example information'
                }
            }
        )
        self.populate_test_event(
            {
                'summary': 'Test Summary',
                'ip': '1.2.3.4',
                'details': {
                    'information': 'Example information'
                }
            }
        )

        self.refresh(self.event_index_name)

        results = query.execute(self.es_client)
        assert results.keys() == ['hits', 'meta', 'aggregations']
        assert results['meta'].keys() == ['timed_out']
        assert results['meta']['timed_out'] is False

        sorted_hits = sorted(results['hits'], key=lambda k: k['_source']['ip'])

        assert len(sorted_hits) == 3

        assert sorted_hits[0].keys() == ['_score', '_type', '_id', '_source', '_index']
        assert type(sorted_hits[0]['_id']) == unicode
        assert sorted_hits[0]['_type'] == 'event'

        assert sorted_hits[0]['_index'] == datetime.now().strftime("events-%Y%m%d")

        assert sorted_hits[0]['_source']['ip'] == '1.2.3.4'
        assert sorted_hits[0]['_source']['summary'] == 'Test Summary'

        assert sorted_hits[0]['_source']['details'].keys() == ['information']
        assert sorted_hits[0]['_source']['details']['information'] == 'Example information'

        assert sorted_hits[1].keys() == ['_score', '_type', '_id', '_source', '_index']
        assert type(sorted_hits[1]['_id']) == unicode
        assert sorted_hits[1]['_type'] == 'event'

        assert sorted_hits[1]['_index'] == datetime.now().strftime("events-%Y%m%d")

        assert sorted_hits[1]['_source']['ip'] == '1.2.3.4'
        assert sorted_hits[1]['_source']['summary'] == 'Test Summary'

        assert sorted_hits[1]['_source']['details'].keys() == ['information']
        assert sorted_hits[1]['_source']['details']['information'] == 'Example information'

        assert type(sorted_hits[2]['_id']) == unicode
        assert sorted_hits[2]['_type'] == 'event'

        assert sorted_hits[2]['_index'] == datetime.now().strftime("events-%Y%m%d")

        assert sorted_hits[2]['_source']['ip'] == '127.0.0.1'
        assert sorted_hits[2]['_source']['summary'] == 'Test Summary'

        assert sorted_hits[2]['_source']['details'].keys() == ['information']
        assert sorted_hits[2]['_source']['details']['information'] == 'Example information'

        assert results['aggregations'].keys() == ['ip']

        assert results['aggregations']['ip'].keys() == ['terms']

        assert len(results['aggregations']['ip']['terms']) == 2

        results['aggregations']['ip']['terms'].sort()
        assert results['aggregations']['ip']['terms'][0]['count'] == 1
        assert results['aggregations']['ip']['terms'][0]['key'] == '127.0.0.1'

        assert results['aggregations']['ip']['terms'][1]['count'] == 2
        assert results['aggregations']['ip']['terms'][1]['key'] == '1.2.3.4'