Golang models.ParsePoints函数代码示例

func BenchmarkParsePointsTagsSorted2(b *testing.B) {
	line := `cpu,host=serverA,region=us-west value=1i 1000000000`
	for i := 0; i < b.N; i++ {
		models.ParsePoints([]byte(line))
		b.SetBytes(int64(len(line)))
	}
}

func TestParsePointToString(t *testing.T) {
	line := `cpu,host=serverA,region=us-east bool=false,float=11,float2=12.123,int=10i,str="string val" 1000000000`
	pts, err := models.ParsePoints([]byte(line))
	if err != nil {
		t.Fatalf(`ParsePoints() failed. got %s`, err)
	}
	if exp := 1; len(pts) != exp {
		t.Errorf("ParsePoint() len mismatch: got %v, exp %v", len(pts), exp)
	}
	pt := pts[0]

	got := pt.String()
	if line != got {
		t.Errorf("ParsePoint() to string mismatch:\n got %v\n exp %v", got, line)
	}

	pt = models.MustNewPoint("cpu", models.Tags{"host": "serverA", "region": "us-east"},
		models.Fields{"int": 10, "float": float64(11.0), "float2": float64(12.123), "bool": false, "str": "string val"},
		time.Unix(1, 0))

	got = pt.String()
	if line != got {
		t.Errorf("NewPoint() to string mismatch:\n got %v\n exp %v", got, line)
	}
}

func BenchmarkParsePointNoTags(b *testing.B) {
	line := `cpu value=1i 1000000000`
	for i := 0; i < b.N; i++ {
		models.ParsePoints([]byte(line))
		b.SetBytes(int64(len(line)))
	}
}

// parser() reads all incoming messages from the consumer, and parses them into
// influxdb metric points.
func (k *Kafka) parser() {
	for {
		select {
		case <-k.done:
			return
		case err := <-k.errs:
			log.Printf("Kafka Consumer Error: %s\n", err.Error())
		case msg := <-k.in:
			points, err := models.ParsePoints(msg.Value)
			if err != nil {
				log.Printf("Could not parse kafka message: %s, error: %s",
					string(msg.Value), err.Error())
			}

			for _, point := range points {
				select {
				case k.pointChan <- point:
					continue
				default:
					log.Printf("Kafka Consumer buffer is full, dropping a point." +
						" You may want to increase the point_buffer setting")
				}
			}

			if !k.doNotCommitMsgs {
				// TODO(cam) this locking can be removed if this PR gets merged:
				// https://github.com/wvanbergen/kafka/pull/84
				k.Lock()
				k.Consumer.CommitUpto(msg)
				k.Unlock()
			}
		}
	}
}

func (s *Service) processPackets() {
	defer s.wg.Done()

	for p := range s.packets {
		points, err := models.ParsePoints(p)
		if err != nil {
			s.statMap.Add(statPointsParseFail, 1)
			s.Logger.Printf("E! Failed to parse points: %s", err)
			continue
		}

		if err := s.PointsWriter.WritePoints(&cluster.WritePointsRequest{
			Database:         s.config.Database,
			RetentionPolicy:  s.config.RetentionPolicy,
			ConsistencyLevel: cluster.ConsistencyLevelOne,
			Points:           points,
		}); err == nil {
			s.statMap.Add(statPointsTransmitted, int64(len(points)))
		} else {
			s.Logger.Printf("E! failed to write points to database %q: %s", s.config.Database, err)
			s.statMap.Add(statTransmitFail, 1)
		}

		s.statMap.Add(statPointsReceived, int64(len(points)))
	}
}

func BenchmarkParsePointsTagsSorted10(b *testing.B) {
	line := `cpu,env=prod,host=serverA,region=us-west,tag1=value1,tag2=value2,tag3=value3,tag4=value4,tag5=value5,target=servers,zone=1c value=1i 1000000000`
	for i := 0; i < b.N; i++ {
		models.ParsePoints([]byte(line))
		b.SetBytes(int64(len(line)))
	}
}

func (p *Processor) unmarshalWrite(b []byte) (uint64, []models.Point, error) {
	if len(b) < 8 {
		return 0, nil, fmt.Errorf("too short: len = %d", len(b))
	}
	ownerID := binary.BigEndian.Uint64(b[:8])
	points, err := models.ParsePoints(b[8:])
	return ownerID, points, err
}

func BenchmarkParsePointsTagsUnSorted5(b *testing.B) {
	line := `cpu,region=us-west,host=serverA,env=prod,target=servers,zone=1c value=1i 1000000000`
	for i := 0; i < b.N; i++ {
		pt, _ := models.ParsePoints([]byte(line))
		b.SetBytes(int64(len(line)))
		pt[0].Key()
	}
}

func TestStoreEnsureSeriesPersistedInNewShards(t *testing.T) {
	dir, err := ioutil.TempDir("", "store_test")
	if err != nil {
		t.Fatalf("Store.Open() failed to create temp dir: %v", err)
	}
	defer os.RemoveAll(dir)

	s := tsdb.NewStore(dir)
	s.EngineOptions.Config.WALDir = filepath.Join(dir, "wal")
	if err := s.Open(); err != nil {
		t.Fatalf("Store.Open() failed: %v", err)
	}

	if err := s.CreateShard("foo", "default", 1); err != nil {
		t.Fatalf("error creating shard: %v", err)
	}

	p, _ := models.ParsePoints([]byte("cpu val=1"))
	if err := s.WriteToShard(1, p); err != nil {
		t.Fatalf("error writing to shard: %v", err)
	}

	if err := s.CreateShard("foo", "default", 2); err != nil {
		t.Fatalf("error creating shard: %v", err)
	}

	if err := s.WriteToShard(2, p); err != nil {
		t.Fatalf("error writing to shard: %v", err)
	}

	d := s.DatabaseIndex("foo")
	if d == nil {
		t.Fatal("expected to have database index for foo")
	}
	if d.Series("cpu") == nil {
		t.Fatal("expected series cpu to be in the index")
	}

	// delete the shard, close the store and reopen it and confirm the measurement is still there
	s.DeleteShard(1)
	s.Close()

	s = tsdb.NewStore(dir)
	s.EngineOptions.Config.WALDir = filepath.Join(dir, "wal")
	if err := s.Open(); err != nil {
		t.Fatalf("Store.Open() failed: %v", err)
	}

	d = s.DatabaseIndex("foo")
	if d == nil {
		t.Fatal("expected to have database index for foo")
	}
	if d.Series("cpu") == nil {
		t.Fatal("expected series cpu to be in the index")
	}
}

func (w *WriteShardRequest) unmarshalPoints() []models.Point {
	points := make([]models.Point, len(w.pb.GetPoints()))
	for i, p := range w.pb.GetPoints() {
		pt, err := models.ParsePoints(p)
		if err != nil {
			// A error here means that one node parsed the point correctly but sent an
			// unparseable version to another node.  We could log and drop the point and allow
			// anti-entropy to resolve the discrepancy but this shouldn't ever happen.
			panic(fmt.Sprintf("failed to parse point: `%v`: %v", string(p), err))
		}
		points[i] = pt[0]
	}
	return points
}

func TestParsePointKeyUnsorted(t *testing.T) {
	pts, err := models.ParsePoints([]byte("cpu,last=1,first=2 value=1i"))
	if err != nil {
		t.Fatalf(`ParsePoints() failed. got %s`, err)
	}

	if exp := 1; len(pts) != exp {
		t.Errorf("ParsePoint() len mismatch: got %v, exp %v", len(pts), exp)
	}
	pt := pts[0]

	if exp := "cpu,first=2,last=1"; string(pt.Key()) != exp {
		t.Errorf("ParsePoint key not sorted. got %v, exp %v", string(pt.Key()), exp)
	}
}

func emitMetrics(k *Kafka, acc plugins.Accumulator, metricConsumer <-chan []byte) error {
	timeout := time.After(1 * time.Second)

	for {
		select {
		case batch := <-metricConsumer:
			var points []models.Point
			var err error
			if points, err = models.ParsePoints(batch); err != nil {
				return err
			}

			for _, point := range points {
				acc.AddFields(point.Name(), point.Fields(), point.Tags(), point.Time())
			}
		case <-timeout:
			return nil
		}
	}
}

func (s *Service) parser() {
	defer s.wg.Done()

	for {
		select {
		case <-s.done:
			return
		case buf := <-s.parserChan:
			points, err := models.ParsePoints(buf)
			if err != nil {
				s.statMap.Add(statPointsParseFail, 1)
				s.Logger.Printf("Failed to parse points: %s", err)
				continue
			}

			for _, point := range points {
				s.batcher.In() <- point
			}
			s.statMap.Add(statPointsReceived, int64(len(points)))
		}
	}
}

func TestBasicPointGenerator_Generate(t *testing.T) {
	ps, err := basicPG.Generate()
	if err != nil {
		t.Error(err)
	}

	var buf bytes.Buffer

	for p := range ps {
		b := p.Line()

		buf.Write(b)
		buf.Write([]byte("\n"))
	}

	bs := buf.Bytes()
	bs = bs[0 : len(bs)-1]

	_, err = models.ParsePoints(bs)
	if err != nil {
		t.Error(err)
	}
}

func TestParsePointIntsFloats(t *testing.T) {
	pts, err := models.ParsePoints([]byte(`cpu,host=serverA,region=us-east int=10i,float=11.0,float2=12.1 1000000000`))
	if err != nil {
		t.Fatalf(`ParsePoints() failed. got %s`, err)
	}

	if exp := 1; len(pts) != exp {
		t.Errorf("ParsePoint() len mismatch: got %v, exp %v", len(pts), exp)
	}
	pt := pts[0]

	if _, ok := pt.Fields()["int"].(int64); !ok {
		t.Errorf("ParsePoint() int field mismatch: got %T, exp %T", pt.Fields()["int"], int64(10))
	}

	if _, ok := pt.Fields()["float"].(float64); !ok {
		t.Errorf("ParsePoint() float field mismatch: got %T, exp %T", pt.Fields()["float64"], float64(11.0))
	}

	if _, ok := pt.Fields()["float2"].(float64); !ok {
		t.Errorf("ParsePoint() float field mismatch: got %T, exp %T", pt.Fields()["float64"], float64(12.1))
	}
}