Golang core.PodKey函数代码示例

func TestGetLabeledMetrics(t *testing.T) {
	now := time.Now().UTC()
	key := core.PodKey("ns1", "pod1")
	otherKey := core.PodKey("ns1", "other")

	batch1, batch2, batch3 := makeBatches(now, key, otherKey)

	metrics := NewMetricSink(45*time.Second, 120*time.Second, []string{"m1"})
	metrics.ExportData(&batch1)
	metrics.ExportData(&batch2)
	metrics.ExportData(&batch3)

	result := metrics.GetLabeledMetric("somelblmetric", map[string]string{"lbl1": "val1.1", "lbl2": "val2.1"}, []string{key}, now.Add(-120*time.Second), now)

	assert.Equal(t, []core.TimestampedMetricValue{
		{
			Timestamp: now.Add(-20 * time.Second),
			MetricValue: core.MetricValue{
				ValueType:  core.ValueInt64,
				MetricType: core.MetricGauge,
				IntValue:   309,
			},
		},
	}, result[key])
}

func (this *PodBasedEnricher) Process(batch *core.DataBatch) (*core.DataBatch, error) {
	newMs := make(map[string]*core.MetricSet, len(batch.MetricSets))
	for k, v := range batch.MetricSets {
		switch v.Labels[core.LabelMetricSetType.Key] {
		case core.MetricSetTypePod:
			namespace := v.Labels[core.LabelNamespaceName.Key]
			podName := v.Labels[core.LabelPodName.Key]
			pod, err := this.getPod(namespace, podName)
			if err != nil {
				glog.V(3).Infof("Failed to get pod %s from cache: %v", core.PodKey(namespace, podName), err)
				continue
			}
			addPodInfo(k, v, pod, batch, newMs)
		case core.MetricSetTypePodContainer:
			namespace := v.Labels[core.LabelNamespaceName.Key]
			podName := v.Labels[core.LabelPodName.Key]
			pod, err := this.getPod(namespace, podName)
			if err != nil {
				glog.V(3).Infof("Failed to get pod %s from cache: %v", core.PodKey(namespace, podName), err)
				continue
			}
			addContainerInfo(k, v, pod, batch, newMs)
		}
	}
	for k, v := range newMs {
		batch.MetricSets[k] = v
	}
	return batch, nil
}

func TestGetMetrics(t *testing.T) {
	now := time.Now()
	key := core.PodKey("ns1", "pod1")
	otherKey := core.PodKey("ns1", "other")

	batch1, batch2, batch3 := makeBatches(now, key, otherKey)

	metrics := NewMetricSink(45*time.Second, 120*time.Second, []string{"m1"})
	metrics.ExportData(&batch1)
	metrics.ExportData(&batch2)
	metrics.ExportData(&batch3)

	//batch1 is discarded by long store
	result1 := metrics.GetMetric("m1", []string{key}, now.Add(-120*time.Second), now)
	assert.Equal(t, 2, len(result1[key]))
	assert.Equal(t, int64(40), result1[key][0].MetricValue.IntValue)
	assert.Equal(t, int64(20), result1[key][1].MetricValue.IntValue)
	assert.Equal(t, 1, len(metrics.GetMetric("m1", []string{otherKey}, now.Add(-120*time.Second), now)[otherKey]))

	//batch1 is discarded by long store and batch2 doesn't belong to time window
	assert.Equal(t, 1, len(metrics.GetMetric("m1", []string{key}, now.Add(-30*time.Second), now)[key]))

	//batch1 and batch1 are discarded by short store
	assert.Equal(t, 1, len(metrics.GetMetric("m2", []string{key}, now.Add(-120*time.Second), now)[key]))

	//nothing is in time window
	assert.Equal(t, 0, len(metrics.GetMetric("m2", []string{key}, now.Add(-10*time.Second), now)[key]))

	metricNames := metrics.GetMetricNames(key)
	assert.Equal(t, 2, len(metricNames))
	assert.Contains(t, metricNames, "m1")
	assert.Contains(t, metricNames, "m2")
}

func TestClusterAggregate(t *testing.T) {
	batch := core.DataBatch{
		Timestamp: time.Now(),
		MetricSets: map[string]*core.MetricSet{
			core.PodKey("ns1", "pod1"): {
				Labels: map[string]string{
					core.LabelMetricSetType.Key: core.MetricSetTypeNamespace,
					core.LabelNamespaceName.Key: "ns1",
				},
				MetricValues: map[string]core.MetricValue{
					"m1": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   10,
					},
					"m2": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   222,
					},
				},
			},

			core.PodKey("ns1", "pod2"): {
				Labels: map[string]string{
					core.LabelMetricSetType.Key: core.MetricSetTypeNamespace,
					core.LabelNamespaceName.Key: "ns1",
				},
				MetricValues: map[string]core.MetricValue{
					"m1": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   100,
					},
					"m3": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   30,
					},
				},
			},
		},
	}
	processor := ClusterAggregator{
		MetricsToAggregate: []string{"m1", "m3"},
	}
	result, err := processor.Process(&batch)
	assert.NoError(t, err)
	cluster, found := result.MetricSets[core.ClusterKey()]
	assert.True(t, found)

	m1, found := cluster.MetricValues["m1"]
	assert.True(t, found)
	assert.Equal(t, int64(110), m1.IntValue)

	m3, found := cluster.MetricValues["m3"]
	assert.True(t, found)
	assert.Equal(t, int64(30), m3.IntValue)
}

func TestGetNames(t *testing.T) {
	now := time.Now()
	key := core.PodKey("ns1", "pod1")
	otherKey := core.PodKey("ns1", "other")

	batch := core.DataBatch{
		Timestamp: now.Add(-20 * time.Second),
		MetricSets: map[string]*core.MetricSet{
			key: {
				Labels: map[string]string{
					core.LabelMetricSetType.Key: core.MetricSetTypePod,
					core.LabelPodNamespace.Key:  "ns1",
					core.LabelNamespaceName.Key: "ns1",
					core.LabelPodName.Key:       "pod1",
				},
				MetricValues: map[string]core.MetricValue{
					"m1": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   20,
					},
					"m2": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   222,
					},
				},
			},
			otherKey: {
				Labels: map[string]string{
					core.LabelMetricSetType.Key: core.MetricSetTypePod,
					core.LabelPodNamespace.Key:  "ns2",
					core.LabelNamespaceName.Key: "ns2",
					core.LabelPodName.Key:       "pod2",
				},
				MetricValues: map[string]core.MetricValue{
					"m1": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   123,
					},
				},
			},
		},
	}

	metrics := NewMetricSink(45*time.Second, 120*time.Second, []string{"m1"})
	metrics.ExportData(&batch)

	assert.Contains(t, metrics.GetPods(), "ns1/pod1")
	assert.Contains(t, metrics.GetPods(), "ns2/pod2")
	assert.Contains(t, metrics.GetPodsFromNamespace("ns1"), "pod1")
	assert.NotContains(t, metrics.GetPodsFromNamespace("ns1"), "pod2")
	assert.Contains(t, metrics.GetMetricSetKeys(), key)
	assert.Contains(t, metrics.GetMetricSetKeys(), otherKey)
}

func (this *PodAggregator) Process(batch *core.DataBatch) (*core.DataBatch, error) {
	newPods := make(map[string]*core.MetricSet)

	for key, metricSet := range batch.MetricSets {
		if metricSetType, found := metricSet.Labels[core.LabelMetricSetType.Key]; found && metricSetType == core.MetricSetTypePodContainer {
			// Aggregating containers
			podName, found := metricSet.Labels[core.LabelPodName.Key]
			ns, found2 := metricSet.Labels[core.LabelNamespaceName.Key]
			if found && found2 {
				podKey := core.PodKey(ns, podName)
				pod, found := batch.MetricSets[podKey]
				if !found {
					pod, found = newPods[podKey]
					if !found {
						glog.V(2).Infof("Pod not found adding %s", podKey)
						pod = this.podMetricSet(metricSet.Labels)
						newPods[podKey] = pod
					}
				}

				for metricName, metricValue := range metricSet.MetricValues {
					if _, found := this.skippedMetrics[metricName]; found {
						continue
					}

					aggregatedValue, found := pod.MetricValues[metricName]
					if found {
						if aggregatedValue.ValueType != metricValue.ValueType {
							glog.Errorf("PodAggregator: inconsistent type in %s", metricName)
							continue
						}

						switch aggregatedValue.ValueType {
						case core.ValueInt64:
							aggregatedValue.IntValue += metricValue.IntValue
						case core.ValueFloat:
							aggregatedValue.FloatValue += metricValue.FloatValue
						default:
							return nil, fmt.Errorf("PodAggregator: type not supported in %s", metricName)
						}
					} else {
						aggregatedValue = metricValue
					}
					pod.MetricValues[metricName] = aggregatedValue
				}
			} else {
				glog.Errorf("No namespace and/or pod info in container %s: %v", key, metricSet.Labels)
				continue
			}
		}
	}
	for key, val := range newPods {
		batch.MetricSets[key] = val
	}
	return batch, nil
}

func addContainerInfo(key string, containerMs *core.MetricSet, pod *kube_api.Pod, batch *core.DataBatch, newMs map[string]*core.MetricSet) {
	for _, container := range pod.Spec.Containers {
		if key == core.PodContainerKey(pod.Namespace, pod.Name, container.Name) {
			updateContainerResourcesAndLimits(containerMs, container)
			if _, ok := containerMs.Labels[core.LabelContainerBaseImage.Key]; !ok {
				containerMs.Labels[core.LabelContainerBaseImage.Key] = container.Image
			}
			break
		}
	}

	containerMs.Labels[core.LabelPodId.Key] = string(pod.UID)
	containerMs.Labels[core.LabelLabels.Key] = util.LabelsToString(pod.Labels, ",")

	namespace := containerMs.Labels[core.LabelNamespaceName.Key]
	podName := containerMs.Labels[core.LabelPodName.Key]

	podKey := core.PodKey(namespace, podName)
	_, oldfound := batch.MetricSets[podKey]
	if !oldfound {
		_, newfound := newMs[podKey]
		if !newfound {
			glog.V(2).Infof("Pod %s not found, creating a stub", podKey)
			podMs := &core.MetricSet{
				MetricValues: make(map[string]core.MetricValue),
				Labels: map[string]string{
					core.LabelMetricSetType.Key: core.MetricSetTypePod,
					core.LabelNamespaceName.Key: namespace,
					core.LabelPodNamespace.Key:  namespace,
					core.LabelPodName.Key:       podName,
					core.LabelNodename.Key:      containerMs.Labels[core.LabelNodename.Key],
					core.LabelHostname.Key:      containerMs.Labels[core.LabelHostname.Key],
					core.LabelHostID.Key:        containerMs.Labels[core.LabelHostID.Key],
				},
			}
			newMs[podKey] = podMs
			addPodInfo(podKey, podMs, pod, batch, newMs)
		}
	}
}

func (a *Api) podListMetrics(request *restful.Request, response *restful.Response) {
	start, end, err := getStartEndTime(request)
	if err != nil {
		response.WriteError(http.StatusBadRequest, err)
		return
	}
	ns := request.PathParameter("namespace-name")
	keys := []string{}
	metricName := request.PathParameter("metric-name")
	convertedMetricName := convertMetricName(metricName)
	for _, podName := range strings.Split(request.PathParameter("pod-list"), ",") {
		keys = append(keys, core.PodKey(ns, podName))
	}
	metrics := a.metricSink.GetMetric(convertedMetricName, keys, start, end)
	result := types.MetricResultList{
		Items: make([]types.MetricResult, 0, len(keys)),
	}
	for _, key := range keys {
		result.Items = append(result.Items, exportTimestampedMetricValue(metrics[key]))
	}
	response.PrettyPrint(false)
	response.WriteEntity(result)
}

// podMetrics returns a metric timeseries for a metric of the Pod entity.
func (a *Api) podMetrics(request *restful.Request, response *restful.Response) {
	a.processMetricRequest(
		core.PodKey(request.PathParameter("namespace-name"),
			request.PathParameter("pod-name")),
		request, response)
}

func TestPodEnricher(t *testing.T) {
	pod := kube_api.Pod{
		ObjectMeta: kube_api.ObjectMeta{
			Name:      "pod1",
			Namespace: "ns1",
		},
		Spec: kube_api.PodSpec{
			NodeName: "node1",
			Containers: []kube_api.Container{
				{
					Name:  "c1",
					Image: "gcr.io/google_containers/pause:2.0",
					Resources: kube_api.ResourceRequirements{
						Requests: kube_api.ResourceList{
							kube_api.ResourceCPU:    *resource.NewMilliQuantity(100, resource.DecimalSI),
							kube_api.ResourceMemory: *resource.NewQuantity(555, resource.DecimalSI),
						},
					},
				},
				{
					Name:  "nginx",
					Image: "gcr.io/google_containers/pause:2.0",
					Resources: kube_api.ResourceRequirements{
						Requests: kube_api.ResourceList{
							kube_api.ResourceCPU:    *resource.NewMilliQuantity(333, resource.DecimalSI),
							kube_api.ResourceMemory: *resource.NewQuantity(1000, resource.DecimalSI),
						},
						Limits: kube_api.ResourceList{
							kube_api.ResourceCPU:    *resource.NewMilliQuantity(2222, resource.DecimalSI),
							kube_api.ResourceMemory: *resource.NewQuantity(3333, resource.DecimalSI),
						},
					},
				},
			},
		},
	}

	store := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})
	podLister := &cache.StoreToPodLister{Indexer: store}
	podLister.Indexer.Add(&pod)
	podBasedEnricher := PodBasedEnricher{podLister: podLister}

	var err error
	for _, batch := range batches {
		batch, err = podBasedEnricher.Process(batch)
		assert.NoError(t, err)

		podAggregator := PodAggregator{}
		batch, err = podAggregator.Process(batch)
		assert.NoError(t, err)

		podMs, found := batch.MetricSets[core.PodKey("ns1", "pod1")]
		assert.True(t, found)
		checkRequests(t, podMs, 433, 1555)
		checkLimits(t, podMs, 2222, 3333)

		containerMs, found := batch.MetricSets[core.PodContainerKey("ns1", "pod1", "c1")]
		assert.True(t, found)
		checkRequests(t, containerMs, 100, 555)
		checkLimits(t, containerMs, 0, 0)
	}
}

var batches = []*core.DataBatch{
	&core.DataBatch{
		Timestamp: time.Now(),
		MetricSets: map[string]*core.MetricSet{
			core.PodContainerKey("ns1", "pod1", "c1"): {
				Labels: map[string]string{
					core.LabelMetricSetType.Key: core.MetricSetTypePodContainer,
					core.LabelPodName.Key:       "pod1",
					core.LabelNamespaceName.Key: "ns1",
					core.LabelContainerName.Key: "c1",
				},
				MetricValues: map[string]core.MetricValue{},
			},

			core.PodKey("ns1", "pod1"): {
				Labels: map[string]string{
					core.LabelMetricSetType.Key: core.MetricSetTypePod,
					core.LabelPodName.Key:       "pod1",
					core.LabelNamespaceName.Key: "ns1",
				},
				MetricValues: map[string]core.MetricValue{},
			},
		},
	},
	&core.DataBatch{
		Timestamp: time.Now(),
		MetricSets: map[string]*core.MetricSet{
			core.PodContainerKey("ns1", "pod1", "c1"): {
				Labels: map[string]string{
					core.LabelMetricSetType.Key: core.MetricSetTypePodContainer,

func TestDecodeSummaryMetrics(t *testing.T) {
	ms := testingSummaryMetricsSource()
	summary := stats.Summary{
		Node: stats.NodeStats{
			NodeName:  nodeInfo.NodeName,
			StartTime: unversioned.NewTime(startTime),
			CPU:       genTestSummaryCPU(seedNode),
			Memory:    genTestSummaryMemory(seedNode),
			Network:   genTestSummaryNetwork(seedNode),
			SystemContainers: []stats.ContainerStats{
				genTestSummaryContainer(stats.SystemContainerKubelet, seedKubelet),
				genTestSummaryContainer(stats.SystemContainerRuntime, seedRuntime),
				genTestSummaryContainer(stats.SystemContainerMisc, seedMisc),
			},
			Fs: genTestSummaryFsStats(seedNode),
		},
		Pods: []stats.PodStats{{
			PodRef: stats.PodReference{
				Name:      pName0,
				Namespace: namespace0,
			},
			StartTime: unversioned.NewTime(startTime),
			Network:   genTestSummaryNetwork(seedPod0),
			Containers: []stats.ContainerStats{
				genTestSummaryContainer(cName00, seedPod0Container0),
				genTestSummaryContainer(cName01, seedPod0Container1),
			},
		}, {
			PodRef: stats.PodReference{
				Name:      pName1,
				Namespace: namespace0,
			},
			StartTime: unversioned.NewTime(startTime),
			Network:   genTestSummaryNetwork(seedPod1),
			Containers: []stats.ContainerStats{
				genTestSummaryContainer(cName10, seedPod1Container),
			},
			VolumeStats: []stats.VolumeStats{{
				Name:    "A",
				FsStats: *genTestSummaryFsStats(seedPod1),
			}, {
				Name:    "B",
				FsStats: *genTestSummaryFsStats(seedPod1),
			}},
		}, {
			PodRef: stats.PodReference{
				Name:      pName2,
				Namespace: namespace1,
			},
			StartTime: unversioned.NewTime(startTime),
			Network:   genTestSummaryNetwork(seedPod2),
			Containers: []stats.ContainerStats{
				genTestSummaryContainer(cName20, seedPod2Container),
			},
		}},
	}

	containerFs := []string{"/", "logs"}
	expectations := []struct {
		key     string
		setType string
		seed    int64
		cpu     bool
		memory  bool
		network bool
		fs      []string
	}{{
		key:     core.NodeKey(nodeInfo.NodeName),
		setType: core.MetricSetTypeNode,
		seed:    seedNode,
		cpu:     true,
		memory:  true,
		network: true,
		fs:      []string{"/"},
	}, {
		key:     core.NodeContainerKey(nodeInfo.NodeName, "kubelet"),
		setType: core.MetricSetTypeSystemContainer,
		seed:    seedKubelet,
		cpu:     true,
		memory:  true,
	}, {
		key:     core.NodeContainerKey(nodeInfo.NodeName, "docker-daemon"),
		setType: core.MetricSetTypeSystemContainer,
		seed:    seedRuntime,
		cpu:     true,
		memory:  true,
	}, {
		key:     core.NodeContainerKey(nodeInfo.NodeName, "system"),
		setType: core.MetricSetTypeSystemContainer,
		seed:    seedMisc,
		cpu:     true,
		memory:  true,
	}, {
		key:     core.PodKey(namespace0, pName0),
		setType: core.MetricSetTypePod,
		seed:    seedPod0,
		network: true,
	}, {
		key:     core.PodKey(namespace0, pName1),
		setType: core.MetricSetTypePod,
//.........这里部分代码省略.........

func TestPodAggregator(t *testing.T) {
	batch := core.DataBatch{
		Timestamp: time.Now(),
		MetricSets: map[string]*core.MetricSet{
			core.PodContainerKey("ns1", "pod1", "c1"): {
				Labels: map[string]string{
					core.LabelMetricSetType.Key: core.MetricSetTypePodContainer,
					core.LabelPodName.Key:       "pod1",
					core.LabelNamespaceName.Key: "ns1",
				},
				MetricValues: map[string]core.MetricValue{
					"m1": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   10,
					},
					"m2": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   222,
					},
				},
			},

			core.PodContainerKey("ns1", "pod1", "c2"): {
				Labels: map[string]string{
					core.LabelMetricSetType.Key: core.MetricSetTypePodContainer,
					core.LabelPodName.Key:       "pod1",
					core.LabelNamespaceName.Key: "ns1",
				},
				MetricValues: map[string]core.MetricValue{
					"m1": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   100,
					},
					"m3": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   30,
					},
				},
			},
		},
	}
	processor := PodAggregator{}
	result, err := processor.Process(&batch)
	assert.NoError(t, err)
	pod, found := result.MetricSets[core.PodKey("ns1", "pod1")]
	assert.True(t, found)

	m1, found := pod.MetricValues["m1"]
	assert.True(t, found)
	assert.Equal(t, int64(110), m1.IntValue)

	m2, found := pod.MetricValues["m2"]
	assert.True(t, found)
	assert.Equal(t, int64(222), m2.IntValue)

	m3, found := pod.MetricValues["m3"]
	assert.True(t, found)
	assert.Equal(t, int64(30), m3.IntValue)

	labelPodName, found := pod.Labels[core.LabelPodName.Key]
	assert.True(t, found)
	assert.Equal(t, "pod1", labelPodName)

	labelNsName, found := pod.Labels[core.LabelNamespaceName.Key]
	assert.True(t, found)
	assert.Equal(t, "ns1", labelNsName)
}

func TestGetMetrics(t *testing.T) {
	now := time.Now()
	key := core.PodKey("ns1", "pod1")
	otherKey := core.PodKey("ns1", "other")

	batch1 := core.DataBatch{
		Timestamp: now.Add(-180 * time.Second),
		MetricSets: map[string]*core.MetricSet{
			key: {
				Labels: map[string]string{
					core.LabelMetricSetType.Key: core.MetricSetTypePod,
					core.LabelPodNamespace.Key:  "ns1",
				},
				MetricValues: map[string]core.MetricValue{
					"m1": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   60,
					},
					"m2": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   666,
					},
				},
			},
		},
	}

	batch2 := core.DataBatch{
		Timestamp: now.Add(-60 * time.Second),
		MetricSets: map[string]*core.MetricSet{
			key: {
				Labels: map[string]string{
					core.LabelMetricSetType.Key: core.MetricSetTypePod,
					core.LabelPodNamespace.Key:  "ns1",
				},
				MetricValues: map[string]core.MetricValue{
					"m1": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   40,
					},
					"m2": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   444,
					},
				},
			},
		},
	}

	batch3 := core.DataBatch{
		Timestamp: now.Add(-20 * time.Second),
		MetricSets: map[string]*core.MetricSet{
			key: {
				Labels: map[string]string{
					core.LabelMetricSetType.Key: core.MetricSetTypePod,
					core.LabelPodNamespace.Key:  "ns1",
				},
				MetricValues: map[string]core.MetricValue{
					"m1": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   20,
					},
					"m2": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   222,
					},
				},
			},
			otherKey: {
				Labels: map[string]string{
					core.LabelMetricSetType.Key: core.MetricSetTypePod,
					core.LabelPodNamespace.Key:  "ns1",
				},
				MetricValues: map[string]core.MetricValue{
					"m1": {
						ValueType:  core.ValueInt64,
						MetricType: core.MetricGauge,
						IntValue:   123,
					},
				},
			},
		},
	}

	metrics := NewMetricSink(45*time.Second, 120*time.Second, []string{"m1"})
	metrics.ExportData(&batch1)
	metrics.ExportData(&batch2)
	metrics.ExportData(&batch3)

	//batch1 is discarded by long store
	result1 := metrics.GetMetric("m1", []string{key}, now.Add(-120*time.Second), now)
	assert.Equal(t, 2, len(result1[key]))
	assert.Equal(t, 40, result1[key][0].MetricValue.IntValue)
	assert.Equal(t, 20, result1[key][1].MetricValue.IntValue)
//.........这里部分代码省略.........