Python anomaly_detector.AnomalyDetector类代码示例

 def test_algorithm_DefaultDetector(self):
   """
   Test if optional parameter algorithm works as expected.
   """
   detector = AnomalyDetector(self.s1, algorithm_name='default_detector')
   self.assertEqual(detector.get_all_scores().timestamps, self.detector1.get_all_scores().timestamps)
   self.assertEqual(detector.get_all_scores().values, self.detector1.get_all_scores().values)

 def test_algorithm_params(self):
   """
   Test if optional parameter algorithm_params works as expected.
   """
   self.assertRaises(exceptions.InvalidDataFormat, lambda: AnomalyDetector(self.s1, algorithm_name='exp_avg_detector', algorithm_params='0'))
   detector = AnomalyDetector(self.s1, algorithm_name="exp_avg_detector", algorithm_params={'smoothing_factor': 0.3})
   self.assertNotEqual(self.detector1.get_all_scores().values, detector.get_all_scores().values)

 def test_anomaly_threshold(self):
   """
   Test if score_percentile_threshold works as expected.
   """
   detector = AnomalyDetector(self.s1, score_percentile_threshold=0.1, algorithm_name='exp_avg_detector')
   detector1 = AnomalyDetector(self.s1, score_percentile_threshold=0.1, algorithm_name='derivative_detector')
   self.assertNotEqual(detector1.get_anomalies(), detector.get_anomalies())

 def test_threshold(self):
   """
   Test score threshold=0
   """
   detector = AnomalyDetector(self.s1, score_threshold=0)
   self.assertTrue(len(detector.get_anomalies()) == 1)
   self.assertTrue(detector.get_anomalies() is not None)

 def test_score_only(self):
   """
   Test that score_only parameter doesn't give anomalies
   """
   detector = AnomalyDetector(self.s1, score_only=True, algorithm_name='derivative_detector')
   detector2 = AnomalyDetector(self.s1, algorithm_name='derivative_detector')
   self.assertTrue(detector2.get_anomalies() is not None)
   self.assertTrue(len(detector.get_anomalies()) == 0)

 def test_custom_algorithm(self):
   """
   Test passing a custom algorithm class
   """
   detector = AnomalyDetector(self.s1, baseline_time_series=self.s2, algorithm_class=CustomAlgo,
                              algorithm_params={'percent_threshold_upper': 20, 'percent_threshold_lower': -20})
   anomalies = detector.get_anomalies()
   self.assertTrue(anomalies is not None)
   self.assertTrue(len(anomalies) > 0)

 def test_absolute_threshold_algorithm(self):
   """
   Test "absolute threshold" algorithm with a upper and lower threshold of 0.2
   """
   detector = AnomalyDetector(self.s1, algorithm_name='absolute_threshold',
                              algorithm_params={'absolute_threshold_value_upper': 0.2,
                                                'absolute_threshold_value_lower': 0.2})
   anomalies = detector.get_anomalies()
   self.assertTrue(anomalies is not None)
   self.assertTrue(len(anomalies) > 0)
   self.assertRaises(exceptions.RequiredParametersNotPassed,
                     lambda: AnomalyDetector(self.s1, algorithm_name='absolute_threshold'))

 def test_diff_percent_threshold_algorithm(self):
   """
   Test "diff percent threshold" algorithm with a threshold of 20%
   """
   detector = AnomalyDetector(self.s1, baseline_time_series=self.s2, algorithm_name='diff_percent_threshold',
                              algorithm_params={'percent_threshold_upper': 20, 'percent_threshold_lower': -20})
   anomalies = detector.get_anomalies()
   self.assertTrue(anomalies is not None)
   self.assertTrue(len(anomalies) > 0)
   self.assertRaises(exceptions.RequiredParametersNotPassed,
                     lambda: AnomalyDetector(self.s1, baseline_time_series=self.s2,
                                             algorithm_name='diff_percent_threshold'))

 def __init__(self, metrix, related_metrices):
   """
   Initializer
   :param metrix: a TimeSeries, a dictionary or a path to a csv file(str)
   :param list related_metrixes: a list of time series.
   """
   self.metrix = self._load(metrix)
   self.anomaly_detector = AnomalyDetector(metrix)
   self.related_metrices = related_metrices
   self.anomalies = self.anomaly_detector.get_anomalies()
   self._analyze()

  def test_sign_test_algorithm_interface(self):
    """
    Test "sign test" algorithm with a threshold of 0%
    """
    bs = dict()
    bs.update((t, 1) for t in range(1, 30))

    # Simple tests
    algorithm_params = {'percent_threshold_upper': 0,
                        'offset': 2,
                        'scan_window': 24,
                        'confidence': 0.01}
    ts = dict(bs)
    # bigger than 10 percent but below bias
    ts.update((t, 3.1) for t in range(1, 21))

    # first no anomalies
    detector = AnomalyDetector(ts, baseline_time_series=bs, algorithm_name='sign_test',
                               algorithm_params=algorithm_params)
    anomalies = detector.get_anomalies()

    self.assertTrue(anomalies is not None)
    self.assertEquals(len(anomalies), 1)

    # try lower bound
    algorithm_params = {'percent_threshold_lower': 0,
                        'offset': 2,
                        'scan_window': 24,
                        'confidence': 0.01}
    ts = dict(bs)
    # less than baseline plus bias
    ts.update((t, 2.9) for t in range(1, 25))

    # first no anomalies
    detector = AnomalyDetector(ts, baseline_time_series=bs, algorithm_name='sign_test',
                               algorithm_params=algorithm_params)
    anomalies = detector.get_anomalies()

    self.assertTrue(anomalies is not None)
    self.assertEquals(len(anomalies), 1)

    if args.interpolate_period > 0:
        tf.set_interpolate(Interpolate(period={'count': args.interpolate_period, 'unit': TimeUnit.MINUTE},
                                       function=InterpolateFunction.LINEAR))

    query.set_transformation_filter(tf)

    series_list = svc.query(query)
    for series in series_list:
        metric_id = '- %s %s' % (series.metric, print_tags(series.tags))
        log('\t' + metric_id)
        # exclude empty series for specific tags
        if len(series.data) > 2:
            ts = {int(sample.t / 1000): sample.v for sample in series.data}

            detector = AnomalyDetector(ts, score_threshold=args.min_score)

            anomalies = []
            for anomaly in detector.get_anomalies():
                if time.mktime(now.timetuple()) - args.last_hours * 3600 <= anomaly.exact_timestamp:
                    anomalies.append(anomaly)

            if anomalies:
                message.append(metric_id)
                for anomaly in anomalies:
                    t_start, t_end = format_t(anomaly.start_timestamp), format_t(anomaly.end_timestamp)
                    t_exact = format_t(anomaly.exact_timestamp)
                    anomaly_msg = '\tAnomaly from %s to %s with score %s: %s, %s' % (
                        t_start, t_end, anomaly.anomaly_score, t_exact, ts[anomaly.exact_timestamp])
                    message.append(anomaly_msg)

class RCA(object):
  def __init__(self, metrix, related_metrices):
    """
    Initializer
    :param metrix: a TimeSeries, a dictionary or a path to a csv file(str)
    :param list related_metrixes: a list of time series.
    """
    self.metrix = self._load(metrix)
    self.anomaly_detector = AnomalyDetector(metrix)
    self.related_metrices = related_metrices
    self.anomalies = self.anomaly_detector.get_anomalies()
    self._analyze()

  def _load(self, metrix):
    """
    Load time series.
    :param timeseries: a TimeSeries, a dictionary or a path to a csv file(str).
    :return TimeSeries: a TimeSeries object.
    """
    if isinstance(metrix, TimeSeries):
      return metrix
    if isinstance(metrix, dict):
      return TimeSeries(metrix)
    return TimeSeries(utils.read_csv(metrix))

  def _analyze(self):
    """
    Analyzes if a matrix has anomalies.
    If any anomaly is found, determine if the matrix correlates with any other matrixes.
    To be implemented.
    """
    output = defaultdict(list)
    output_by_name = defaultdict(list)
    scores = self.anomaly_detector.get_all_scores()

    if self.anomalies:
      for anomaly in self.anomalies:
        metrix_scores = scores
        start_t, end_t = anomaly.get_time_window()
        t = anomaly.exact_timestamp

        # Compute extended start timestamp and extended end timestamp.
        room = (end_t - start_t) / 2
        if not room:
          room = 30
        extended_start_t = start_t - room
        extended_end_t = end_t + room
        metrix_scores_cropped = metrix_scores.crop(extended_start_t, extended_end_t)

        # Adjust the two timestamps if not enough data points are included.
        while len(metrix_scores_cropped) < 2:
          extended_start_t = extended_start_t - room
          extended_end_t = extended_end_t + room
          metrix_scores_cropped = metrix_scores.crop(extended_start_t, extended_end_t)

        # Correlate with other metrics
        for entry in self.related_metrices:
          try:
            entry_correlation_result = Correlator(self.metrix, entry, time_period=(extended_start_t, extended_end_t),
                                                  use_anomaly_score=True).get_correlation_result()
            record = extended_start_t, extended_end_t, entry_correlation_result.__dict__, entry
            record_by_name = extended_start_t, extended_end_t, entry_correlation_result.__dict__
            output[t].append(record)
            output_by_name[entry].append(record_by_name)
          except exceptions.NotEnoughDataPoints:
            pass

    self.output = output
    self.output_by_name = output_by_name

 def setUp(self):
   self.s1 = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0.5, 5: 1, 6: 1, 7: 1, 8: 0}
   self.s2 = {0: 0, 1: 0.5, 2: 1, 3: 1, 4: 1, 5: 0, 6: 0, 7: 0, 8: 0}
   self.detector1 = AnomalyDetector(self.s1)
   self.detector2 = AnomalyDetector(self.s2)

class TestAnomalyDetector(unittest.TestCase):

  def setUp(self):
    self.s1 = {0: 0, 1: 0, 2: 0, 3: 0, 4: 0.5, 5: 1, 6: 1, 7: 1, 8: 0}
    self.s2 = {0: 0, 1: 0.5, 2: 1, 3: 1, 4: 1, 5: 0, 6: 0, 7: 0, 8: 0}
    self.detector1 = AnomalyDetector(self.s1)
    self.detector2 = AnomalyDetector(self.s2)

  def test_threshold(self):
    detector = AnomalyDetector(self.s1, score_threshold=0)
    self.assertTrue(len(detector.get_anomalies()) == 1)
    self.assertTrue(detector.get_anomalies() is not None)

  def test_score_only(self):
    detector = AnomalyDetector(self.s1, score_only=True, algorithm_name='derivative_detector')
    detector2 = AnomalyDetector(self.s1, algorithm_name='derivative_detector')
    self.assertTrue(detector2.get_anomalies() is not None)
    self.assertTrue(detector.get_anomalies() is None)

  def test_get_all_scores(self):
    """
    Test if function get_all_scores works as expected.
    """
    self.assertTrue(isinstance(self.detector1.get_all_scores(), TimeSeries))
    self.assertEqual(len(self.detector1.get_all_scores()), len(self.detector1.time_series))

  def test_get_anomalies(self):
    """
    Test if anomaly is found as expected.
    """
    self.assertTrue(self.detector1.get_anomalies() is not None)

  def test_algorithm_DefaultDetector(self):
    """
    Test if optional parameter algorithm works as expected.
    """
    detector = AnomalyDetector(self.s1, algorithm_name='default_detector')
    self.assertEqual(detector.get_all_scores().timestamps, self.detector1.get_all_scores().timestamps)
    self.assertEqual(detector.get_all_scores().values, self.detector1.get_all_scores().values)

  def test_algorithm(self):
    """
    Test if exception AlgorithmNotFound is raised as expected.
    """
    self.assertRaises(exceptions.AlgorithmNotFound, lambda: AnomalyDetector(self.s1, algorithm_name='NotValidAlgorithm'))

  def test_algorithm_params(self):
    """
    Test if optional parameter algorithm_params works as expected.
    """
    self.assertRaises(exceptions.InvalidDataFormat, lambda: AnomalyDetector(self.s1, algorithm_name='exp_avg_detector', algorithm_params='0'))
    detector = AnomalyDetector(self.s1, algorithm_name="exp_avg_detector", algorithm_params={'smoothing_factor': 0.3})
    self.assertNotEqual(self.detector1.get_all_scores().values, detector.get_all_scores().values)

  def test_anomaly_threshold(self):
    """
    Test if score_percentile_threshold works as expected.
    """
    detector = AnomalyDetector(self.s1, score_percentile_threshold=0.1, algorithm_name='exp_avg_detector')
    detector1 = AnomalyDetector(self.s1, score_percentile_threshold=0.1, algorithm_name='derivative_detector')
    self.assertNotEqual(detector1.get_anomalies(), detector.get_anomalies())

  def test_sign_test_algorithm_with_shift(self):
    """
    Test "sign test" algorithm with a threshold of 20%
    """
    bs = dict()
    bs.update({t: 1 for t in range(1, 100)})

    # Simple tests
    algorithm_params = {'percent_threshold_upper': 10,
                      'shift': 1,
                      'scan_window': 24,
                      'confidence': 0.01}
    ts = dict(bs)
    # bigger than 10 percent but below bias
    ts.update({t: 1.2 for t in range(10, 34)})

    # first no anomalies
    detector = AnomalyDetector(ts, baseline_time_series=bs, algorithm_name='sign_test',
                               algorithm_params=algorithm_params)
    anomalies = detector.get_anomalies()

    self.assertTrue(anomalies is not None)
    self.assertEquals(len(anomalies), 0)

    # Next one anomaly exactly equal to scan window
    # uses bias
    ts.update({t: 2.100001 for t in range(10, 34)})
    detector = AnomalyDetector(ts, baseline_time_series=bs, algorithm_name='sign_test',
                               algorithm_params=algorithm_params)
    anomalies = detector.get_anomalies()

    self.assertTrue(anomalies is not None)
    self.assertEquals(len(anomalies), 1)
    anomaly = anomalies[0]
    s, e = anomaly.get_time_window()

    # note the anomaly is larger than scan window
    self.assertEquals(s, 4)
    self.assertEquals(e, 39)

    # score should be roughly 98.5
    self.assertGreater(anomaly.anomaly_score, 98)
    self.assertLess(anomaly.anomaly_score, 99)

    # anomaly below baseline but not below baseline with shift
    algorithm_params = {'percent_threshold_lower': -20,
                      'shift': -0.1,
                      'scan_window': 24,
                      'confidence': 0.01}
    ts.update({t: 0.799999 for t in range(10, 34)})
    # no anomalies
    detector = AnomalyDetector(ts, baseline_time_series=bs, algorithm_name='sign_test',
                               algorithm_params=algorithm_params)
    anomalies = detector.get_anomalies()

    self.assertTrue(anomalies is not None)
    self.assertEquals(len(anomalies), 0)

    # lower the time series by 0.1
    ts.update({t: 0.699999 for t in range(10, 34)})

    detector = AnomalyDetector(ts, baseline_time_series=bs, algorithm_name='sign_test',
                               algorithm_params=algorithm_params)
    anomalies = detector.get_anomalies()
    self.assertEquals(len(anomalies), 1)
    anomaly = anomalies[0]
    s, e = anomaly.get_time_window()
    self.assertEquals(s, 4)
    self.assertEquals(e, 39)

    # score should be roughly 98.5
    self.assertGreater(anomaly.anomaly_score, 98)
    self.assertLess(anomaly.anomaly_score, 99)