Python histogram.HdrHistogram类代码示例

    def encode_bins(self, p_output):
        p_output = json.loads(p_output)
        p_output['jobs'][0].pop('trim')
        test_list = ['read', 'write']

        for test in test_list:
            histogram = HdrHistogram(1, 5 * 3600 * 1000, 3)
            clat = p_output['jobs'][0][test]['clat']['bins']
            total_buckets = clat['FIO_IO_U_PLAT_NR']
            grp_msb_bits = clat['FIO_IO_U_PLAT_BITS']
            buckets_per_grp = clat['FIO_IO_U_PLAT_VAL']

            for bucket in xrange(total_buckets):
                if clat[str(bucket)]:
                    grp = bucket / buckets_per_grp
                    subbucket = bucket % buckets_per_grp
                    if grp == 0:
                        val = subbucket - 1
                    else:
                        base = 2 ** (grp_msb_bits + grp - 1)
                        val = int(base + (base / buckets_per_grp) * (subbucket - 0.5))
                    histogram.record_value(val, clat[str(bucket)])

            p_output['jobs'][0][test]['clat']['hist'] = histogram.encode()
            p_output['jobs'][0][test]['clat'].pop('bins')
            p_output['jobs'][0][test]['clat'].pop('percentile')

        return json.dumps(p_output)

def test_mean_stddev():
    # fill up a histogram with the values in the list
    histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT)
    for value in VALUES_LIST:
        histogram.record_value(value)
    assert(histogram.get_mean_value() == 2000.5)
    assert(histogram.get_stddev() == 1000.5)

    def consolidate_results(results):
        all_res = {'tool': 'wrk2'}
        total_count = len(results)
        if not total_count:
            return all_res

        for key in ['http_rps', 'http_total_req', 'http_sock_err',
                    'http_sock_timeout', 'http_throughput_kbytes']:
            all_res[key] = 0
            for item in results:
                if (key in item['results']):
                    all_res[key] += item['results'][key]
            all_res[key] = int(all_res[key])

        if 'latency_stats' in results[0]['results']:
            # for item in results:
            #     print item['results']['latency_stats']
            all_res['latency_stats'] = []
            histogram = HdrHistogram(1, 24 * 3600 * 1000 * 1000, 2)
            for item in results:
                histogram.decode_and_add(item['results']['latency_stats'])
            perc_list = [50, 75, 90, 99, 99.9, 99.99, 99.999]
            latency_dict = histogram.get_percentile_to_value_dict(perc_list)
            for key, value in latency_dict.iteritems():
                all_res['latency_stats'].append([key, value])
            all_res['latency_stats'].sort()

        return all_res

    def consolidate_results(results):
        err_flag = False
        all_res = {'tool': 'wrk2'}
        total_count = len(results)
        if not total_count:
            return all_res

        for key in ['http_rps', 'http_total_req', 'http_sock_err',
                    'http_sock_timeout', 'http_throughput_kbytes']:
            all_res[key] = 0
            for item in results:
                all_res[key] += item['results'].get(key, 0)
            all_res[key] = int(all_res[key])

        if 'latency_stats' in results[0]['results']:
            # for item in results:
            #     print item['results']['latency_stats']
            all_res['latency_stats'] = []
            histogram = HdrHistogram(1, 24 * 3600 * 1000 * 1000, 2)
            for item in results:
                if 'latency_stats' in item['results']:
                    histogram.decode_and_add(item['results']['latency_stats'])
                else:
                    err_flag = True
            perc_list = [50, 75, 90, 99, 99.9, 99.99, 99.999]
            latency_dict = histogram.get_percentile_to_value_dict(perc_list)
            for key, value in latency_dict.iteritems():
                all_res['latency_stats'].append([key, value])
            all_res['latency_stats'].sort()

        if err_flag:
            LOG.warning('Unable to find latency_stats from the result dictionary, this '
                        'may indicate that the test application on VM exited abnormally.')

        return all_res

def test_highest_equivalent_value():
    histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT)
    assert 8183 * 1024 + 1023 == histogram.get_highest_equivalent_value(8180 * 1024)
    assert 8191 * 1024 + 1023 == histogram.get_highest_equivalent_value(8191 * 1024)
    assert 8199 * 1024 + 1023 == histogram.get_highest_equivalent_value(8193 * 1024)
    assert 9999 * 1024 + 1023 == histogram.get_highest_equivalent_value(9995 * 1024)
    assert 10007 * 1024 + 1023 == histogram.get_highest_equivalent_value(10007 * 1024)
    assert 10015 * 1024 + 1023 == histogram.get_highest_equivalent_value(10008 * 1024)

def test_scaled_highest_equiv_value():
    histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT)
    assert 8183 == histogram.get_highest_equivalent_value(8180)
    assert 8191 == histogram.get_highest_equivalent_value(8191)
    assert 8199 == histogram.get_highest_equivalent_value(8193)
    assert 9999 == histogram.get_highest_equivalent_value(9995)
    assert 10007 == histogram.get_highest_equivalent_value(10007)
    assert 10015 == histogram.get_highest_equivalent_value(10008)

def check_cod_perf():
    histogram = HdrHistogram(LOWEST, WRK2_MAX_LATENCY, 2)
    fill_start_index = (20 * histogram.counts_len) // 100
    fill_to_index = fill_start_index + (30 * histogram.counts_len) // 100
    fill_hist_counts(histogram, fill_to_index, fill_start_index)

    # encode 1000 times
    start = datetime.datetime.now()
    for _ in range(1000):
        histogram.encode()
    delta = datetime.datetime.now() - start
    print(delta)

def check_hist_encode(word_size,
                      digits,
                      expected_compressed_length,
                      fill_start_percent,
                      fill_count_percent):
    histogram = HdrHistogram(LOWEST, WRK2_MAX_LATENCY, digits,
                             word_size=word_size)
    if fill_count_percent:
        fill_start_index = (fill_start_percent * histogram.counts_len) // 100
        fill_to_index = fill_start_index + (fill_count_percent * histogram.counts_len) // 100
        fill_hist_counts(histogram, fill_to_index, fill_start_index)
    b64 = histogram.encode()
    assert(len(b64) == expected_compressed_length)

def check_dec_perf():
    histogram = HdrHistogram(LOWEST, WRK2_MAX_LATENCY, 2)
    fill_start_index = (20 * histogram.counts_len) // 100
    fill_to_index = fill_start_index + (30 * histogram.counts_len) // 100
    fill_hist_counts(histogram, fill_to_index, fill_start_index)
    b64 = histogram.encode()

    # decode and add to self 1000 times
    start = datetime.datetime.now()
    for _ in range(1000):
        histogram.decode_and_add(b64)
    delta = datetime.datetime.now() - start
    print(delta)

def test_hist_codec_partial():
    histogram = HdrHistogram(LOWEST, WRK2_MAX_LATENCY, SIGNIFICANT)

    partial_histogram = HdrHistogram(LOWEST, WRK2_MAX_LATENCY, SIGNIFICANT)

    # put some known numbers in the first half buckets
    half_count = partial_histogram.counts_len
    fill_hist_counts(partial_histogram, half_count)
    encoded = partial_histogram.encode()
    histogram.decode_and_add(encoded)

    # now verify that the partial counters are identical to the original
    check_hist_counts(histogram, half_count, multiplier=1)
    check_hist_counts(histogram, histogram.counts_len, start=half_count + 1, multiplier=0)

def test_jHiccup_v2_log():
    accumulated_histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT)
    for checklist in JHICCUP_CHECKLISTS:
        accumulated_histogram.reset()
        log_reader = HistogramLogReader(JHICCUP_V2_LOG_NAME, accumulated_histogram)

        histogram_count = 0
        total_count = 0
        target_numbers = checklist.pop('target')
        while 1:
            decoded_histogram = log_reader.get_next_interval_histogram(**checklist)
            if not decoded_histogram:
                break
            histogram_count += 1
            total_count += decoded_histogram.get_total_count()
            accumulated_histogram.add(decoded_histogram)
            # These logs use 8 byte counters
            assert(decoded_histogram.get_word_size() == 8)
        for statement in target_numbers:
            assert(eval(statement) == target_numbers[statement])

        log_reader.close()

def test_hdr_interop():
    # decode and add the encoded histograms
    histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT)
    corrected_histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT)
    histogram.decode_and_add(ENCODE_SAMPLES_HDRHISTOGRAM_C[0])
    corrected_histogram.decode_and_add(ENCODE_SAMPLES_HDRHISTOGRAM_C[1])

    # check the percentiles. min, max values match
    check_percentiles(histogram, corrected_histogram)

def check_hist_codec_b64(word_size, b64_wrap):
    histogram = HdrHistogram(LOWEST, WRK2_MAX_LATENCY, SIGNIFICANT,
                             b64_wrap=b64_wrap,
                             word_size=word_size)
    # encode with all zero counters
    encoded = histogram.encode()
    # add back same histogram
    histogram.decode_and_add(encoded)
    # counters should remain zero
    check_hist_counts(histogram, histogram.counts_len, multiplier=0)
    # fill up the histogram
    fill_hist_counts(histogram, histogram.counts_len)
    encoded = histogram.encode()
    histogram.decode_and_add(encoded)
    check_hist_counts(histogram, histogram.counts_len, multiplier=2)

#.........这里部分代码省略.........
            with the recorded interval to the [latest, optional] start time
            found in the log. The start time is indicated in the log with
            a "#[StartTime: " followed by the start time in seconds.

        Params:
            dest_histogram if None, created a new histogram, else adds
                           the new interval histogram to it
            range_start_time_sec The absolute or relative start of the expected
                                 time range, in seconds.
            range_start_time_sec The absolute or relative end of the expected
                                  time range, in seconds.
            absolute Defines if the passed range is absolute or relative

        Return:
            Returns an histogram object if an interval line was found with an
            associated start timestamp value that falls between start_time_sec and
            end_time_sec,
            or null if no such interval line is found.
            Upon encountering any unexpected format errors in reading the next
            interval from the file, this method will return None.

            The histogram returned will have it's timestamp set to the absolute
            timestamp calculated from adding the interval's indicated timestamp
            value to the latest [optional] start time found in the log.

        Exceptions:
            ValueError if there is a syntax error in one of the float fields
        '''
        while 1:
            line = self.input_file.readline()
            if not line:
                return None
            if line[0] == '#':
                match_res = re_start_time.match(line)
                if match_res:
                    self.start_time_sec = float(match_res.group(1))
                    self.observed_start_time = True
                    continue
                match_res = re_base_time.match(line)
                if match_res:
                    self.base_time = float(match_res.group(1))
                    self.observed_base_time = True
                    continue

            match_res = re_histogram_interval.match(line)
            if not match_res:
                # probably a legend line that starts with "\"StartTimestamp"
                continue
            # Decode: startTimestamp, intervalLength, maxTime, histogramPayload
            # Timestamp is expected to be in seconds
            log_time_stamp_in_sec = float(match_res.group(1))
            interval_length_sec = float(match_res.group(2))
            cpayload = match_res.group(4)

            if not self.observed_start_time:
                # No explicit start time noted. Use 1st observed time:
                self.start_time_sec = log_time_stamp_in_sec
                self.observed_start_time = True

            if not self.observed_base_time:
                # No explicit base time noted.
                # Deduce from 1st observed time (compared to start time):
                if log_time_stamp_in_sec < self.start_time_sec - (365 * 24 * 3600.0):
                    # Criteria Note: if log timestamp is more than a year in
                    # the past (compared to StartTime),
                    # we assume that timestamps in the log are not absolute
                    self.base_time_sec = self.start_time_sec
                else:
                    # Timestamps are absolute
                    self.base_time_sec = 0.0
                self.observed_base_time = True

            absolute_start_time_stamp_sec = \
                log_time_stamp_in_sec + self.base_time_sec
            offset_start_time_stamp_sec = \
                absolute_start_time_stamp_sec - self.start_time_sec

            # Timestamp length is expect to be in seconds
            absolute_end_time_stamp_sec = \
                absolute_start_time_stamp_sec + interval_length_sec

            if absolute:
                start_time_stamp_to_check_range_on = absolute_start_time_stamp_sec
            else:
                start_time_stamp_to_check_range_on = offset_start_time_stamp_sec

            if start_time_stamp_to_check_range_on < range_start_time_sec:
                continue

            if start_time_stamp_to_check_range_on > range_end_time_sec:
                return None
            if dest_histogram:
                # add the interval histogram to the destination histogram
                histogram = dest_histogram
                histogram.decode_and_add(cpayload)
            else:
                histogram = HdrHistogram.decode(cpayload)
                histogram.set_start_time_stamp(absolute_start_time_stamp_sec * 1000.0)
                histogram.set_end_time_stamp(absolute_end_time_stamp_sec * 1000.0)
            return histogram

def load_corrected_histogram():
    histogram = HdrHistogram(LOWEST, HIGHEST, SIGNIFICANT)
    # record this value with a count of 10,000
    histogram.record_corrected_value(1000, INTERVAL, 10000)
    histogram.record_corrected_value(100000000, INTERVAL)
    return histogram