Golang pbutil.ReadDelimited函数代码示例

func fetchMetricFamilies(url string, ch chan<- *dto.MetricFamily) {
	defer close(ch)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatalf("creating GET request for URL %q failed: %s", url, err)
	}
	req.Header.Add("Accept", acceptHeader)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatalf("executing GET request for URL %q failed: %s", url, err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		log.Fatalf("GET request for URL %q returned HTTP status %s", url, resp.Status)
	}

	mediatype, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
	if err == nil && mediatype == "application/vnd.google.protobuf" &&
		params["encoding"] == "delimited" &&
		params["proto"] == "io.prometheus.client.MetricFamily" {
		for {
			mf := &dto.MetricFamily{}
			if _, err = pbutil.ReadDelimited(resp.Body, mf); err != nil {
				if err == io.EOF {
					break
				}
				log.Fatalln("reading metric family protocol buffer failed:", err)
			}
			ch <- mf
		}
	} else {
		// We could do further content-type checks here, but the
		// fallback for now will anyway be the text format
		// version 0.0.4, so just go for it and see if it works.
		var parser expfmt.TextParser
		metricFamilies, err := parser.TextToMetricFamilies(resp.Body)
		if err != nil {
			log.Fatalln("reading text format failed:", err)
		}
		for _, mf := range metricFamilies {
			ch <- mf
		}
	}
}

// BenchmarkParseProtoGzip is like BenchmarkParseProto above, but parses gzipped
// protobuf format.
func BenchmarkParseProtoGzip(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("testdata/protobuf.gz")
	if err != nil {
		b.Fatal(err)
	}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		family := &dto.MetricFamily{}
		in, err := gzip.NewReader(bytes.NewReader(data))
		if err != nil {
			b.Fatal(err)
		}
		for {
			family.Reset()
			if _, err := pbutil.ReadDelimited(in, family); err != nil {
				if err == io.EOF {
					break
				}
				b.Fatal(err)
			}
		}
	}
}

// loadSnapshot loads a snapshot generated by Snapshot() into the state.
// Any previous state is wiped.
func (s *Silences) loadSnapshot(r io.Reader) error {
	st := gossipData{}

	s.mtx.Lock()
	defer s.mtx.Unlock()

	for {
		var sil pb.MeshSilence
		if _, err := pbutil.ReadDelimited(r, &sil); err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		st[sil.Silence.Id] = &sil
		_, err := s.mc.Get(sil.Silence)
		if err != nil {
			return err
		}
	}

	s.st = st

	return nil
}

// Decode implements the Decoder interface.
func (d *protoDecoder) Decode(v *dto.MetricFamily) error {
	_, err := pbutil.ReadDelimited(d.r, v)
	if err != nil {
		return err
	}
	if !model.IsValidMetricName(model.LabelValue(v.GetName())) {
		return fmt.Errorf("invalid metric name %q", v.GetName())
	}
	for _, m := range v.GetMetric() {
		if m == nil {
			continue
		}
		for _, l := range m.GetLabel() {
			if l == nil {
				continue
			}
			if !model.LabelValue(l.GetValue()).IsValid() {
				return fmt.Errorf("invalid label value %q", l.GetValue())
			}
			if !model.LabelName(l.GetName()).IsValid() {
				return fmt.Errorf("invalid label name %q", l.GetName())
			}
		}
	}
	return nil
}

func (u *delimitedUnmarshaller) Unmarshal(reader io.Reader, entry *Entry) error {
	pbEntry := &protologpb.Entry{}
	if _, err := pbutil.ReadDelimited(reader, pbEntry); err != nil {
		return err
	}
	iEntry, err := pbEntryToEntry(pbEntry)
	if err != nil {
		return err
	}
	*entry = *iEntry
	return nil
}

func decodeGossipData(msg []byte) (gossipData, error) {
	gd := gossipData{}
	rd := bytes.NewReader(msg)

	for {
		var s pb.MeshSilence
		if _, err := pbutil.ReadDelimited(rd, &s); err != nil {
			if err == io.EOF {
				break
			}
			return gd, err
		}
		gd[s.Silence.Id] = &s
	}
	return gd, nil
}

func decodeGossipData(msg []byte) (gossipData, error) {
	gd := gossipData{}
	rd := bytes.NewReader(msg)

	for {
		var e pb.MeshEntry
		if _, err := pbutil.ReadDelimited(rd, &e); err != nil {
			if err == io.EOF {
				break
			}
			return gd, err
		}
		gd[stateKey(e.Entry.GroupKey, e.Entry.Receiver)] = &e
	}

	return gd, nil
}

func (m *metricFamilyProcessor) ProcessSingle(i io.Reader, out Ingester, o *ProcessOptions) error {
	family := &dto.MetricFamily{}

	for {
		family.Reset()

		if _, err := pbutil.ReadDelimited(i, family); err != nil {
			if err == io.EOF {
				return nil
			}
			return err
		}
		if err := extractMetricFamily(out, o, family); err != nil {
			return err
		}
	}
}

// loadSnapshot loads a snapshot generated by Snapshot() into the state.
func (l *nlog) loadSnapshot(r io.Reader) error {
	l.mtx.Lock()
	defer l.mtx.Unlock()

	st := gossipData{}

	for {
		var e pb.MeshEntry
		if _, err := pbutil.ReadDelimited(r, &e); err != nil {
			if err == io.EOF {
				break
			}
			return err
		}
		st[stateKey(e.Entry.GroupKey, e.Entry.Receiver)] = &e
	}
	l.st = st

	return nil
}

// BenchmarkParseProtoMap is like BenchmarkParseProto but DOES put the parsed
// metric family DTOs into a map. This is not happening during Prometheus
// ingestion. It is just here to measure the overhead of that map creation and
// separate it from the overhead of the text format parsing.
func BenchmarkParseProtoMap(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("testdata/protobuf")
	if err != nil {
		b.Fatal(err)
	}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		families := map[string]*dto.MetricFamily{}
		in := bytes.NewReader(data)
		for {
			family := &dto.MetricFamily{}
			if _, err := pbutil.ReadDelimited(in, family); err != nil {
				if err == io.EOF {
					break
				}
				b.Fatal(err)
			}
			families[family.GetName()] = family
		}
	}
}

// Decode implements the Decoder interface.
func (d *protoDecoder) Decode(v *dto.MetricFamily) error {
	_, err := pbutil.ReadDelimited(d.r, v)
	return err
}

// Push returns an http.Handler which accepts samples over HTTP and stores them
// in the MetricStore. If replace is true, all metrics for the job and instance
// given by the request are deleted before new ones are stored.
//
// The returned handler is already instrumented for Prometheus.
func Push(
	ms storage.MetricStore, replace bool,
) func(http.ResponseWriter, *http.Request, httprouter.Params) {
	var ps httprouter.Params
	var mtx sync.Mutex // Protects ps.

	instrumentedHandlerFunc := prometheus.InstrumentHandlerFunc(
		"push",
		func(w http.ResponseWriter, r *http.Request) {
			job := ps.ByName("job")
			labelsString := ps.ByName("labels")
			mtx.Unlock()

			labels, err := splitLabels(labelsString)
			if err != nil {
				http.Error(w, err.Error(), http.StatusBadRequest)
				return
			}
			if job == "" {
				http.Error(w, "job name is required", http.StatusBadRequest)
				return
			}
			labels["job"] = job

			if replace {
				ms.SubmitWriteRequest(storage.WriteRequest{
					Labels:    labels,
					Timestamp: time.Now(),
				})
			}

			var metricFamilies map[string]*dto.MetricFamily
			ctMediatype, ctParams, ctErr := mime.ParseMediaType(r.Header.Get("Content-Type"))
			if ctErr == nil && ctMediatype == "application/vnd.google.protobuf" &&
				ctParams["encoding"] == "delimited" &&
				ctParams["proto"] == "io.prometheus.client.MetricFamily" {
				metricFamilies = map[string]*dto.MetricFamily{}
				for {
					mf := &dto.MetricFamily{}
					if _, err = pbutil.ReadDelimited(r.Body, mf); err != nil {
						if err == io.EOF {
							err = nil
						}
						break
					}
					metricFamilies[mf.GetName()] = mf
				}
			} else {
				// We could do further content-type checks here, but the
				// fallback for now will anyway be the text format
				// version 0.0.4, so just go for it and see if it works.
				var parser text.Parser
				metricFamilies, err = parser.TextToMetricFamilies(r.Body)
			}
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			sanitizeLabels(metricFamilies, labels)
			ms.SubmitWriteRequest(storage.WriteRequest{
				Labels:         labels,
				Timestamp:      time.Now(),
				MetricFamilies: metricFamilies,
			})
			w.WriteHeader(http.StatusAccepted)
		},
	)

	return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		mtx.Lock()
		ps = params
		instrumentedHandlerFunc(w, r)
	}
}

// Parse returns a slice of Metrics from a text representation of a
// metrics
func (p *PrometheusParser) Parse(buf []byte) ([]telegraf.Metric, error) {
	var metrics []telegraf.Metric
	var parser expfmt.TextParser
	// parse even if the buffer begins with a newline
	buf = bytes.TrimPrefix(buf, []byte("\n"))
	// Read raw data
	buffer := bytes.NewBuffer(buf)
	reader := bufio.NewReader(buffer)

	// Get format
	mediatype, params, err := mime.ParseMediaType(p.PromFormat["Content-Type"])
	// Prepare output
	metricFamilies := make(map[string]*dto.MetricFamily)
	if err == nil && mediatype == "application/vnd.google.protobuf" &&
		params["encoding"] == "delimited" &&
		params["proto"] == "io.prometheus.client.MetricFamily" {
		for {
			metricFamily := &dto.MetricFamily{}
			if _, err = pbutil.ReadDelimited(reader, metricFamily); err != nil {
				if err == io.EOF {
					break
				}
				return nil, fmt.Errorf("reading metric family protocol buffer failed: %s", err)
			}
			metricFamilies[metricFamily.GetName()] = metricFamily
		}
	} else {
		metricFamilies, err = parser.TextToMetricFamilies(reader)
		if err != nil {
			return nil, fmt.Errorf("reading text format failed: %s", err)
		}
		// read metrics
		for metricName, mf := range metricFamilies {
			for _, m := range mf.Metric {
				// reading tags
				tags := makeLabels(m)
				/*
					for key, value := range p.DefaultTags {
						tags[key] = value
					}
				*/
				// reading fields
				fields := make(map[string]interface{})
				if mf.GetType() == dto.MetricType_SUMMARY {
					// summary metric
					fields = makeQuantiles(m)
					fields["count"] = float64(m.GetSummary().GetSampleCount())
					fields["sum"] = float64(m.GetSummary().GetSampleSum())
				} else if mf.GetType() == dto.MetricType_HISTOGRAM {
					// historgram metric
					fields = makeBuckets(m)
					fields["count"] = float64(m.GetHistogram().GetSampleCount())
					fields["sum"] = float64(m.GetHistogram().GetSampleSum())

				} else {
					// standard metric
					fields = getNameAndValue(m)
				}
				// converting to telegraf metric
				if len(fields) > 0 {
					var t time.Time
					if m.TimestampMs != nil && *m.TimestampMs > 0 {
						t = time.Unix(0, *m.TimestampMs*1000000)
					} else {
						t = time.Now()
					}
					metric, err := telegraf.NewMetric(metricName, tags, fields, t)
					if err == nil {
						metrics = append(metrics, metric)
					}
				}
			}
		}
	}
	return metrics, err
}

func (u *unmarshaller) Unmarshal(reader io.Reader, entry *Entry) error {
	_, err := pbutil.ReadDelimited(reader, entry)
	return err
}