Python ruletypes.NewTermsRule类代码示例

def test_new_term_with_terms():
    rules = {'fields': ['a'],
             'timestamp_field': '@timestamp',
             'es_host': 'example.com', 'es_port': 10, 'index': 'logstash', 'query_key': 'a',
             'window_step_size': {'days': 2}}
    mock_res = {'aggregations': {'filtered': {'values': {'buckets': [{'key': 'key1', 'doc_count': 1},
                                                                     {'key': 'key2', 'doc_count': 5}]}}}}

    with mock.patch('elastalert.ruletypes.elasticsearch_client') as mock_es:
        mock_es.return_value = mock.Mock()
        mock_es.return_value.search.return_value = mock_res
        mock_es.return_value.info.return_value = {'version': {'number': '2.x.x'}}
        rule = NewTermsRule(rules)

        # Only 15 queries because of custom step size
        assert rule.es.search.call_count == 15

    # Key1 and key2 shouldn't cause a match
    terms = {ts_now(): [{'key': 'key1', 'doc_count': 1},
                        {'key': 'key2', 'doc_count': 1}]}
    rule.add_terms_data(terms)
    assert rule.matches == []

    # Key3 causes an alert for field a
    terms = {ts_now(): [{'key': 'key3', 'doc_count': 1}]}
    rule.add_terms_data(terms)
    assert len(rule.matches) == 1
    assert rule.matches[0]['new_field'] == 'a'
    assert rule.matches[0]['a'] == 'key3'
    rule.matches = []

    # Key3 doesn't cause another alert
    terms = {ts_now(): [{'key': 'key3', 'doc_count': 1}]}
    rule.add_terms_data(terms)
    assert rule.matches == []

def test_new_term_with_terms():
    rules = {'fields': ['a'],
             'timestamp_field': '@timestamp',
             'es_host': 'example.com', 'es_port': 10, 'index': 'logstash', 'query_key': 'a'}
    mock_res = {'aggregations': {'filtered': {'values': {'buckets': [{'key': 'key1', 'doc_count': 1},
                                                                     {'key': 'key2', 'doc_count': 5}]}}}}

    with mock.patch('elastalert.ruletypes.Elasticsearch') as mock_es:
        mock_es.return_value = mock.Mock()
        mock_es.return_value.search.return_value = mock_res
        rule = NewTermsRule(rules)

        assert rule.es.search.call_count == 1

    # Key1 and key2 shouldn't cause a match
    terms = {ts_now(): [{'key': 'key1', 'doc_count': 1},
                        {'key': 'key2', 'doc_count': 1}]}
    rule.add_terms_data(terms)
    assert rule.matches == []

    # Key3 causes an alert for field a
    terms = {ts_now(): [{'key': 'key3', 'doc_count': 1}]}
    rule.add_terms_data(terms)
    assert len(rule.matches) == 1
    assert rule.matches[0]['new_field'] == 'a'
    assert rule.matches[0]['a'] == 'key3'
    rule.matches = []

    # Key3 doesn't cause another alert
    terms = {ts_now(): [{'key': 'key3', 'doc_count': 1}]}
    rule.add_terms_data(terms)
    assert rule.matches == []

def test_new_term_nested_field():

    rules = {'fields': ['a', 'b.c'],
             'timestamp_field': '@timestamp',
             'es_host': 'example.com', 'es_port': 10, 'index': 'logstash'}
    mock_res = {'aggregations': {'filtered': {'values': {'buckets': [{'key': 'key1', 'doc_count': 1},
                                                                     {'key': 'key2', 'doc_count': 5}]}}}}
    with mock.patch('elastalert.ruletypes.Elasticsearch') as mock_es:
        mock_es.return_value = mock.Mock()
        mock_es.return_value.search.return_value = mock_res
        rule = NewTermsRule(rules)

        assert rule.es.search.call_count == 2

    # Key3 causes an alert for nested field b.c
    rule.add_data([{'@timestamp': ts_now(), 'b': {'c': 'key3'}}])
    assert len(rule.matches) == 1
    assert rule.matches[0]['new_field'] == 'b.c'
    assert rule.matches[0]['b']['c'] == 'key3'
    rule.matches = []

def test_new_term():
    rules = {'fields': ['a', 'b'],
             'timestamp_field': '@timestamp',
             'es_host': 'example.com', 'es_port': 10, 'index': 'logstash'}
    mock_res = {'aggregations': {'filtered': {'values': {'buckets': [{'key': 'key1', 'doc_count': 1},
                                                                     {'key': 'key2', 'doc_count': 5}]}}}}

    with mock.patch('elastalert.ruletypes.Elasticsearch') as mock_es:
        mock_es.return_value = mock.Mock()
        mock_es.return_value.search.return_value = mock_res
        rule = NewTermsRule(rules)

        assert rule.es.search.call_count == 2

    # Key1 and key2 shouldn't cause a match
    rule.add_data([{'@timestamp': ts_now(), 'a': 'key1', 'b': 'key2'}])
    assert rule.matches == []

    # Neither will missing values
    rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])
    assert rule.matches == []

    # Key3 causes an alert for field b
    rule.add_data([{'@timestamp': ts_now(), 'a': 'key2', 'b': 'key3'}])
    assert len(rule.matches) == 1
    assert rule.matches[0]['new_field'] == 'b'
    assert rule.matches[0]['b'] == 'key3'
    rule.matches = []

    # Key3 doesn't cause another alert for field b
    rule.add_data([{'@timestamp': ts_now(), 'a': 'key2', 'b': 'key3'}])
    assert rule.matches == []

    # Missing_field
    rules['alert_on_missing_field'] = True
    with mock.patch('elastalert.ruletypes.Elasticsearch') as mock_es:
        mock_es.return_value = mock.Mock()
        mock_es.return_value.search.return_value = mock_res
        rule = NewTermsRule(rules)
    rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])
    assert len(rule.matches) == 1
    assert rule.matches[0]['missing_field'] == 'b'

def test_new_term_with_composite_fields():
    rules = {'fields': [['a', 'b', 'c'], ['d', 'e.f']],
             'timestamp_field': '@timestamp',
             'es_host': 'example.com', 'es_port': 10, 'index': 'logstash'}

    mock_res = {
        'aggregations': {
            'filtered': {
                'values': {
                    'buckets': [
                        {
                            'key': 'key1',
                            'doc_count': 5,
                            'values': {
                                'buckets': [
                                    {
                                        'key': 'key2',
                                        'doc_count': 5,
                                        'values': {
                                            'buckets': [
                                                {
                                                    'key': 'key3',
                                                    'doc_count': 3,
                                                },
                                                {
                                                    'key': 'key4',
                                                    'doc_count': 2,
                                                },
                                            ]
                                        }
                                    }
                                ]
                            }
                        }
                    ]
                }
            }
        }
    }

    with mock.patch('elastalert.ruletypes.elasticsearch_client') as mock_es:
        mock_es.return_value = mock.Mock()
        mock_es.return_value.search.return_value = mock_res
        mock_es.return_value.info.return_value = {'version': {'number': '2.x.x'}}
        rule = NewTermsRule(rules)

        assert rule.es.search.call_count == 60

    # key3 already exists, and thus shouldn't cause a match
    rule.add_data([{'@timestamp': ts_now(), 'a': 'key1', 'b': 'key2', 'c': 'key3'}])
    assert rule.matches == []

    # key5 causes an alert for composite field [a, b, c]
    rule.add_data([{'@timestamp': ts_now(), 'a': 'key1', 'b': 'key2', 'c': 'key5'}])
    assert len(rule.matches) == 1
    assert rule.matches[0]['new_field'] == ('a', 'b', 'c')
    assert rule.matches[0]['a'] == 'key1'
    assert rule.matches[0]['b'] == 'key2'
    assert rule.matches[0]['c'] == 'key5'
    rule.matches = []

    # New values in other fields that are not part of the composite key should not cause an alert
    rule.add_data([{'@timestamp': ts_now(), 'a': 'key1', 'b': 'key2', 'c': 'key4', 'd': 'unrelated_value'}])
    assert len(rule.matches) == 0
    rule.matches = []

    # Verify nested fields work properly
    # Key6 causes an alert for nested field e.f
    rule.add_data([{'@timestamp': ts_now(), 'd': 'key4', 'e': {'f': 'key6'}}])
    assert len(rule.matches) == 1
    assert rule.matches[0]['new_field'] == ('d', 'e.f')
    assert rule.matches[0]['d'] == 'key4'
    assert rule.matches[0]['e']['f'] == 'key6'
    rule.matches = []

    # Missing_fields
    rules['alert_on_missing_field'] = True
    with mock.patch('elastalert.ruletypes.elasticsearch_client') as mock_es:
        mock_es.return_value = mock.Mock()
        mock_es.return_value.search.return_value = mock_res
        mock_es.return_value.info.return_value = {'version': {'number': '2.x.x'}}
        rule = NewTermsRule(rules)
    rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])
    assert len(rule.matches) == 2
    # This means that any one of the three n composite fields were not present
    assert rule.matches[0]['missing_field'] == ('a', 'b', 'c')
    assert rule.matches[1]['missing_field'] == ('d', 'e.f')

def test_new_term():
    rules = {'fields': ['a', 'b'],
             'timestamp_field': '@timestamp',
             'es_host': 'example.com', 'es_port': 10, 'index': 'logstash'}
    mock_res = {'aggregations': {'filtered': {'values': {'buckets': [{'key': 'key1', 'doc_count': 1},
                                                                     {'key': 'key2', 'doc_count': 5}]}}}}

    with mock.patch('elastalert.ruletypes.elasticsearch_client') as mock_es:
        mock_es.return_value = mock.Mock()
        mock_es.return_value.search.return_value = mock_res
        mock_es.return_value.info.return_value = {'version': {'number': '2.x.x'}}
        call_args = []

        # search is called with a mutable dict containing timestamps, this is required to test
        def record_args(*args, **kwargs):
            call_args.append((copy.deepcopy(args), copy.deepcopy(kwargs)))
            return mock_res

        mock_es.return_value.search.side_effect = record_args
        rule = NewTermsRule(rules)

    # 30 day default range, 1 day default step, times 2 fields
    assert rule.es.search.call_count == 60

    # Assert that all calls have the proper ordering of time ranges
    old_ts = '2010-01-01T00:00:00Z'
    old_field = ''
    for call in call_args:
        field = call[1]['body']['aggs']['filtered']['aggs']['values']['terms']['field']
        if old_field != field:
            old_field = field
            old_ts = '2010-01-01T00:00:00Z'
        gte = call[1]['body']['aggs']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['gte']
        assert gte > old_ts
        lt = call[1]['body']['aggs']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['lt']
        assert lt > gte
        old_ts = gte

    # Key1 and key2 shouldn't cause a match
    rule.add_data([{'@timestamp': ts_now(), 'a': 'key1', 'b': 'key2'}])
    assert rule.matches == []

    # Neither will missing values
    rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])
    assert rule.matches == []

    # Key3 causes an alert for field b
    rule.add_data([{'@timestamp': ts_now(), 'a': 'key2', 'b': 'key3'}])
    assert len(rule.matches) == 1
    assert rule.matches[0]['new_field'] == 'b'
    assert rule.matches[0]['b'] == 'key3'
    rule.matches = []

    # Key3 doesn't cause another alert for field b
    rule.add_data([{'@timestamp': ts_now(), 'a': 'key2', 'b': 'key3'}])
    assert rule.matches == []

    # Missing_field
    rules['alert_on_missing_field'] = True
    with mock.patch('elastalert.ruletypes.elasticsearch_client') as mock_es:
        mock_es.return_value = mock.Mock()
        mock_es.return_value.search.return_value = mock_res
        mock_es.return_value.info.return_value = {'version': {'number': '2.x.x'}}
        rule = NewTermsRule(rules)
    rule.add_data([{'@timestamp': ts_now(), 'a': 'key2'}])
    assert len(rule.matches) == 1
    assert rule.matches[0]['missing_field'] == 'b'

def test_new_term():
    rules = {
        "fields": ["a", "b"],
        "timestamp_field": "@timestamp",
        "es_host": "example.com",
        "es_port": 10,
        "index": "logstash",
    }
    mock_res = {
        "aggregations": {
            "filtered": {"values": {"buckets": [{"key": "key1", "doc_count": 1}, {"key": "key2", "doc_count": 5}]}}
        }
    }

    with mock.patch("elastalert.ruletypes.Elasticsearch") as mock_es:
        mock_es.return_value = mock.Mock()
        mock_es.return_value.search.return_value = mock_res
        rule = NewTermsRule(rules)

        assert rule.es.search.call_count == 2

    # Key1 and key2 shouldn't cause a match
    rule.add_data([{"@timestamp": ts_now(), "a": "key1", "b": "key2"}])
    assert rule.matches == []

    # Neither will missing values
    rule.add_data([{"@timestamp": ts_now(), "a": "key2"}])
    assert rule.matches == []

    # Key3 causes an alert for field b
    rule.add_data([{"@timestamp": ts_now(), "a": "key2", "b": "key3"}])
    assert len(rule.matches) == 1
    assert rule.matches[0]["new_field"] == "b"
    assert rule.matches[0]["b"] == "key3"
    rule.matches = []

    # Key3 doesn't cause another alert for field b
    rule.add_data([{"@timestamp": ts_now(), "a": "key2", "b": "key3"}])
    assert rule.matches == []

    # Missing_field
    rules["alert_on_missing_field"] = True
    with mock.patch("elastalert.ruletypes.Elasticsearch") as mock_es:
        mock_es.return_value = mock.Mock()
        mock_es.return_value.search.return_value = mock_res
        rule = NewTermsRule(rules)
    rule.add_data([{"@timestamp": ts_now(), "a": "key2"}])
    assert len(rule.matches) == 1
    assert rule.matches[0]["missing_field"] == "b"