Golang dptest.NewBasicSink函数代码示例

func TestNew(t *testing.T) {
	ctx := context.Background()
	sendTo1 := dptest.NewBasicSink()
	sendTo2 := dptest.NewBasicSink()
	demux := New([]dpsink.Sink{sendTo1, sendTo2})

	pts := []*datapoint.Datapoint{dptest.DP(), dptest.DP()}
	es := []*event.Event{dptest.E(), dptest.E()}
	ctx2, _ := context.WithTimeout(ctx, time.Millisecond)
	assert.Error(t, demux.AddDatapoints(ctx2, pts))
	assert.Error(t, demux.AddEvents(ctx2, es))
	assert.NoError(t, demux.AddDatapoints(context.Background(), []*datapoint.Datapoint{}))
	assert.NoError(t, demux.AddEvents(context.Background(), []*event.Event{}))
}

func TestListenerLoader(t *testing.T) {
	listenFrom := &config.ListenFrom{
		ListenAddr:           workarounds.GolangDoesnotAllowPointerToStringLiteral("127.0.0.1:0"),
		ServerAcceptDeadline: workarounds.GolangDoesnotAllowPointerToTimeLiteral(time.Millisecond),
	}
	ctx := context.Background()
	forwardTo := dptest.NewBasicSink()
	listener, err := ListenerLoader(ctx, forwardTo, listenFrom)
	defer listener.Close()
	assert.Equal(t, nil, err, "Should be ok to make")
	defer listener.Close()
	listeningDialAddress := fmt.Sprintf("127.0.0.1:%d", nettest.TCPPort(listener.psocket))
	assert.Equal(t, numStats, len(listener.Stats()), "Should have no stats")
	assert.NoError(t, err, "Should be ok to make")

	conn, err := net.Dial("tcp", listeningDialAddress)
	assert.NoError(t, err, "Should be ok to make")
	assert.Equal(t, int64(0), listener.stats.invalidDatapoints)
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "%s %d %d\n\nINVALIDLINE", "ametric", 2, 2)
	_, err = buf.WriteTo(conn)
	conn.Close()
	assert.Equal(t, nil, err, "Should be ok to write")
	dp := forwardTo.Next()
	assert.Equal(t, "ametric", dp.Metric, "Should be metric")
	i := dp.Value.(datapoint.IntValue).Int()
	assert.Equal(t, int64(2), i, "Should get 2")

	for atomic.LoadInt64(&listener.stats.retriedListenErrors) == 0 {
		time.Sleep(time.Millisecond)
	}
	assert.Equal(t, int64(1), atomic.LoadInt64(&listener.stats.invalidDatapoints))
}

func TestCounterSinkEvent(t *testing.T) {
	es := []*event.Event{
		{},
		{},
	}
	ctx := context.Background()
	bs := dptest.NewBasicSink()
	count := &Counter{}
	middleSink := NextWrap(count)(bs)
	go func() {
		// Allow time for us to get in the middle of a call
		time.Sleep(time.Millisecond)
		assert.Equal(t, int64(1), atomic.LoadInt64(&count.CallsInFlight), "After a sleep, should be in flight")
		datas := <-bs.EventsChan
		assert.Equal(t, 2, len(datas), "Original datas should be sent")
	}()
	middleSink.AddEvents(ctx, es)
	assert.Equal(t, int64(0), atomic.LoadInt64(&count.CallsInFlight), "Call is finished")
	assert.Equal(t, int64(0), atomic.LoadInt64(&count.TotalProcessErrors), "No errors so far (see above)")
	assert.Equal(t, numTests, len(count.Stats(map[string]string{})), "Just checking stats len()")

	bs.RetError(errors.New("nope"))
	middleSink.AddEvents(ctx, es)
	assert.Equal(t, int64(1), atomic.LoadInt64(&count.TotalProcessErrors), "Error should be sent through")
}

// TODO figure out why this test is flaky, should be > 2, but change to >= 2 so it passes
func TestBufferedForwarderBasicEvent(t *testing.T) {
	ctx := context.Background()
	config := Config{
		BufferSize:         210,
		MaxTotalDatapoints: 1000,
		MaxTotalEvents:     1000,
		NumDrainingThreads: 1,
		MaxDrainSize:       1000,
	}
	sendTo := dptest.NewBasicSink()
	bf := NewBufferedForwarder(ctx, config, sendTo)
	defer bf.Close()
	assert.NoError(t, bf.AddEvents(ctx, []*event.Event{}))
	time.Sleep(time.Millisecond)
	for i := 0; i < 100; i++ {
		datas := []*event.Event{
			dptest.E(),
			dptest.E(),
		}
		assert.NoError(t, bf.AddEvents(ctx, datas))
		if i == 0 {
			seen := <-sendTo.EventsChan
			assert.Equal(t, 2, len(seen), "The first send should eventually come back with the first two events")
		}
	}
	// Wait for more events
	seen := <-sendTo.EventsChan
	assert.True(t, len(seen) >= 2, fmt.Sprintf("Events should buffer: %d", len(seen)))
	assert.Equal(t, numStats, len(bf.Stats(map[string]string{})), "Checking returned stats size")
}

func TestCollectDListenerWithQueryParams(t *testing.T) {
	jsonBody := testCollectdBody

	sendTo := dptest.NewBasicSink()
	ctx := context.Background()
	c := JSONDecoder{
		SendTo: sendTo,
	}

	req, _ := http.NewRequest("POST", "http://127.0.0.1:8081/post-collectd?sfxdim_foo=bar&sfxdim_zam=narf&sfxdim_empty=&pleaseignore=true", bytes.NewBuffer([]byte(jsonBody)))
	req.Header.Set("Content-Type", "application/json")

	loadExpectedDims := map[string]string{"foo": "bar", "zam": "narf", "plugin": "load", "host": "i-b13d1e5f"}
	memoryExpectedDims := map[string]string{"host": "i-b13d1e5f", "plugin": "memory", "dsname": "value", "foo": "bar", "zam": "narf"}
	dfComplexExpectedDims := map[string]string{"plugin": "df", "plugin_instance": "dev", "dsname": "value", "foo": "bar", "zam": "narf", "host": "i-b13d1e5f"}
	parsedInstanceExpectedDims := map[string]string{"foo": "bar", "zam": "narf", "host": "mwp-signalbox", "f": "x", "plugin_instance": "analytics", "k1": "v1", "k2": "v2", "dsname": "value", "plugin": "tail"}
	eventExpectedDims := map[string]string{"foo": "bar", "host": "mwp-signalbox", "plugin": "my_plugin", "f": "x", "plugin_instance": "my_plugin_instance", "k": "v", "zam": "narf"}

	go func() {
		dps := <-sendTo.PointsChan
		assert.Equal(t, "load.shortterm", dps[0].Metric, "Metric not named correctly")
		assert.Equal(t, loadExpectedDims, dps[0].Dimensions, "Dimensions not set correctly")

		assert.Equal(t, "load.midterm", dps[1].Metric, "Metric not named correctly")
		assert.Equal(t, loadExpectedDims, dps[1].Dimensions, "Dimensions not set correctly")

		assert.Equal(t, "load.longterm", dps[2].Metric, "Metric not named correctly")
		assert.Equal(t, loadExpectedDims, dps[2].Dimensions, "Dimensions not set correctly")

		assert.Equal(t, "memory.used", dps[3].Metric, "Metric not named correctly")
		assert.Equal(t, memoryExpectedDims, dps[3].Dimensions, "Dimensions not set correctly")

		assert.Equal(t, "df_complex.free", dps[4].Metric, "Metric not named correctly")
		assert.Equal(t, dfComplexExpectedDims, dps[4].Dimensions, "Dimensions not set correctly")

		assert.Equal(t, "memory.old_gen_end", dps[5].Metric, "Metric not named correctly")
		assert.Equal(t, parsedInstanceExpectedDims, dps[5].Dimensions, "Dimensions not set correctly")

		assert.Equal(t, "memory.total_heap_space", dps[6].Metric, "Metric not named correctly")
		assert.Equal(t, parsedInstanceExpectedDims, dps[6].Dimensions, "Dimensions not set correctly")

		events := <-sendTo.EventsChan
		assert.Equal(t, "imanotify.notify_instance", events[0].EventType, "Metric not named correctly")
		assert.Equal(t, eventExpectedDims, events[0].Dimensions, "Dimensions not set correctly")
	}()
	resp := httptest.NewRecorder()
	c.ServeHTTPC(ctx, resp, req)
	assert.Equal(t, resp.Code, http.StatusOK, "Request should work")

	req, _ = http.NewRequest("POST", "http://127.0.0.1:8081/post-collectd", bytes.NewBuffer([]byte(`invalidjson`)))
	req.Header.Set("Content-Type", "application/json")

	resp = httptest.NewRecorder()
	c.ServeHTTPC(ctx, resp, req)

	assert.Equal(t, c.TotalBlankDims, int64(1))

	assert.Equal(t, http.StatusBadRequest, resp.Code, "Request should work")

}

func TestStatDrainingThreadCancel(t *testing.T) {
	testSink := dptest.NewBasicSink()
	ctx, cancel := context.WithCancel(context.Background())
	drainer := NewDrainingThread(time.Hour, []dpsink.Sink{testSink}, []Keeper{&statKeeper{}}, ctx)
	cancel()
	assert.Equal(t, ctx.Err(), drainer.start())
}

func TestCarbonHandleConnection(t *testing.T) {
	log.Info("START TestCarbonHandleConnection")
	defer log.Info("END   TestCarbonHandleConnection")
	listenFrom := &config.ListenFrom{
		ListenAddr: workarounds.GolangDoesnotAllowPointerToStringLiteral("localhost:0"),
	}

	ctx := context.Background()
	forwardTo := dptest.NewBasicSink()
	listener, err := ListenerLoader(ctx, forwardTo, listenFrom)
	defer listener.Close()

	listeningDialAddress := fmt.Sprintf("localhost:%d", nettest.TCPPort(listener.psocket))

	conn, err := net.Dial("tcp", listeningDialAddress)
	assert.NoError(t, err)
	conn.Close()
	assert.Error(t, listener.handleConnection(conn))

	conn, err = net.Dial("tcp", listeningDialAddress)
	assert.NoError(t, err)
	waitChan := make(chan struct{})
	go func() {
		time.Sleep(time.Millisecond * 10)
		assert.NoError(t, conn.Close())
		close(waitChan)
	}()
	<-waitChan

	for atomic.LoadInt64(&listener.stats.totalEOFCloses) == 0 {
		time.Sleep(time.Millisecond)
	}
}

func TestStatDrainingThreadSend(t *testing.T) {
	testSink := dptest.NewBasicSink()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	drainer := NewDrainingThread(time.Millisecond, []dpsink.Sink{testSink}, []Keeper{&statKeeper{}}, ctx)
	assert.Equal(t, 1, len(drainer.Stats()))
	<-testSink.PointsChan
}

func TestInvalidListen(t *testing.T) {
	listenFrom := &config.ListenFrom{
		ListenAddr: workarounds.GolangDoesnotAllowPointerToStringLiteral("127.0.0.1:999999"),
	}
	sendTo := dptest.NewBasicSink()
	ctx := context.Background()
	_, err := ListenerLoader(ctx, sendTo, listenFrom)
	assert.Error(t, err)
}

func TestCarbonInvalidListenerLoader(t *testing.T) {
	ctx := context.Background()
	listenFrom := &config.ListenFrom{
		ListenAddr: workarounds.GolangDoesnotAllowPointerToStringLiteral("127.0.0.1:999999"),
	}
	sendTo := dptest.NewBasicSink()
	_, err := ListenerLoader(ctx, sendTo, listenFrom)
	assert.NotEqual(t, nil, err, "Should get an error making")
}

func TestCollectDListener(t *testing.T) {
	jsonBody := testCollectdBody

	sendTo := dptest.NewBasicSink()
	ctx := context.Background()
	listenFrom := &config.ListenFrom{
		Dimensions: map[string]string{"hello": "world"},
	}
	collectdListener, err := ListenerLoader(ctx, sendTo, listenFrom)
	defer collectdListener.Close()
	assert.Nil(t, err)
	assert.NotNil(t, collectdListener)

	req, _ := http.NewRequest("POST", "http://127.0.0.1:8081/post-collectd", bytes.NewBuffer([]byte(jsonBody)))
	req.Header.Set("Content-Type", "application/json")
	client := &http.Client{}
	go func() {
		dps := <-sendTo.PointsChan
		assert.Equal(t, len(dps), 8)
		assert.Equal(t, "load.shortterm", dps[0].Metric, "Metric not named correctly")
		assert.Equal(t, "load.midterm", dps[1].Metric, "Metric not named correctly")
		assert.Equal(t, "load.longterm", dps[2].Metric, "Metric not named correctly")
		assert.Equal(t, "memory.used", dps[3].Metric, "Metric not named correctly")
		assert.Equal(t, "df_complex.free", dps[4].Metric, "Metric not named correctly")
		assert.Equal(t, "memory.old_gen_end", dps[5].Metric, "Metric not named correctly")
		assert.Equal(t, "memory.total_heap_space", dps[6].Metric, "Metric not named correctly")

		assert.Equal(t, "gauge.page.loadtime", dps[7].Metric, "Metric not named correctly")
		assert.Equal(t, map[string]string{"hello": "world", "dsname": "value", "plugin": "dogstatsd", "env": "dev", "k1": "v1", "host": "some-host"}, dps[7].Dimensions, "Dimensions not parsed correctly")
		events := <-sendTo.EventsChan
		assert.Equal(t, len(events), 2)
		assert.Equal(t, "imanotify.notify_instance", events[0].EventType, "Event not named correctly")
		assert.Equal(t, "counter.exception", events[1].EventType, "Event not named correctly")
	}()
	resp, err := client.Do(req)
	if err != nil {
		fmt.Println(err.Error())
		assert.Fail(t, "Err should be nil")
	}
	assert.Equal(t, http.StatusOK, resp.StatusCode, "Request should work")

	assert.Equal(t, 12, len(collectdListener.Stats()), "Request should work")

	req, _ = http.NewRequest("POST", "http://127.0.0.1:8081/post-collectd", bytes.NewBuffer([]byte(`invalidjson`)))
	req.Header.Set("Content-Type", "application/json")
	resp, err = client.Do(req)
	assert.Nil(t, err)
	assert.Equal(t, http.StatusBadRequest, resp.StatusCode, "Request should work")

	req, _ = http.NewRequest("POST", "http://127.0.0.1:8081/post-collectd", bytes.NewBuffer([]byte(jsonBody)))
	req.Header.Set("Content-Type", "application/plaintext")
	resp, err = client.Do(req)
	assert.Nil(t, err)
	assert.Equal(t, http.StatusNotFound, resp.StatusCode, "Request should work (Plaintext not supported)")

}

func TestCarbonInvalidCarbonDeconstructorListenerLoader(t *testing.T) {
	listenFrom := &config.ListenFrom{
		ListenAddr:          workarounds.GolangDoesnotAllowPointerToStringLiteral("127.0.0.1:12247"),
		MetricDeconstructor: workarounds.GolangDoesnotAllowPointerToStringLiteral("UNKNOWN"),
	}
	ctx := context.Background()
	forwardTo := dptest.NewBasicSink()
	_, err := ListenerLoader(ctx, forwardTo, listenFrom)
	assert.NotEqual(t, nil, err, "Should get an error making")
}

func TestCreation(t *testing.T) {
	listenFrom := config.ListenFrom{}
	listenFrom.ListenAddr = workarounds.GolangDoesnotAllowPointerToStringLiteral("127.0.0.1:0")
	forwardTo := dptest.NewBasicSink()
	ctx := context.Background()
	l, err := ListenerLoader(ctx, forwardTo, &listenFrom)
	defer l.Close()
	assert.Equal(t, nil, err, "Expect no error")
	assert.Equal(t, numStats, len(l.Stats()), "Expect no stats")
	forwarder, err := NewForwarder("127.0.0.1", nettest.TCPPort(l.psocket), time.Second, []string{"zzfirst"}, 10)
	defer forwarder.Close()
	assert.Equal(t, nil, err, "Expect no error")
	assert.Equal(t, 1, len(forwarder.pool.conns))
	timeToSend := time.Now().Round(time.Second)
	dpSent := dptest.DP()
	dpSent.Timestamp = timeToSend
	log.Info("Sending a dp")
	forwarder.AddDatapoints(ctx, []*datapoint.Datapoint{dpSent})
	log.Info("Looking for DP back")
	dpSeen := forwardTo.Next()

	assert.Equal(t, "randtest."+dpSent.Metric, dpSeen.Metric, "Expect metric back")
	assert.Equal(t, dpSent.Timestamp, dpSeen.Timestamp, "Expect metric back")

	// Test creating a new connection if pool is empty
	for forwarder.pool.Get() != nil {
	}

	forwarder.AddDatapoints(ctx, []*datapoint.Datapoint{dpSent})
	dpSeen = forwardTo.Next()
	assert.Equal(t, "randtest."+dpSent.Metric, dpSeen.Metric, "Expect metric back")
	assert.Equal(t, dpSent.Timestamp, dpSeen.Timestamp, "Expect metric back")
	//
	// Test creation error if pool is empty
	for forwarder.pool.Get() != nil {
	}

	forwarder.dialer = func(network, address string, timeout time.Duration) (net.Conn, error) {
		return nil, errors.New("nope")
	}
	assert.Error(t, forwarder.AddDatapoints(ctx, []*datapoint.Datapoint{dpSeen}))

	forwarder.dialer = net.DialTimeout
	assert.NoError(t, forwarder.AddDatapoints(ctx, []*datapoint.Datapoint{dpSeen}), "Should get the conn back")

}

func BenchmarkCollectdListener(b *testing.B) {
	bytes := int64(0)
	smallCollectdBody := `[
    {
        "dsnames": [
            "shortterm"
        ],
        "dstypes": [
            "gauge"
        ],
        "host": "i-b13d1e5f",
        "interval": 10.0,
        "plugin": "load",
        "plugin_instance": "",
        "time": 1415062577.4960001,
        "type": "load",
        "type_instance": "",
        "values": [
            0.76000000000000001
        ]
    }]`

	sendTo := dptest.NewBasicSink()
	sendTo.PointsChan = make(chan []*datapoint.Datapoint, 2)
	ctx := context.Background()
	c := JSONDecoder{
		SendTo: sendTo,
	}

	b.ReportAllocs()
	b.StopTimer()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		writter := httptest.NewRecorder()
		body := strings.NewReader(smallCollectdBody)
		req, _ := http.NewRequest("GET", "http://example.com/collectd", body)
		req.Header.Add("Content-type", "application/json")
		b.StartTimer()
		c.ServeHTTPC(ctx, writter, req)
		b.StopTimer()
		bytes += int64(len(testCollectdBody))
		item := <-sendTo.PointsChan
		assert.Equal(b, 1, len(item))
	}
	b.SetBytes(bytes)
}

func TestFailureInRead(t *testing.T) {
	jsonBody := testCollectdBody

	sendTo := dptest.NewBasicSink()
	sendTo.RetError(errors.New("error"))
	ctx := context.Background()
	listenFrom := &config.ListenFrom{}
	collectdListener, err := ListenerLoader(ctx, sendTo, listenFrom)
	defer collectdListener.Close()
	assert.Nil(t, err)
	assert.NotNil(t, collectdListener)

	req, _ := http.NewRequest("POST", "http://127.0.0.1:8081/post-collectd", bytes.NewBuffer([]byte(jsonBody)))
	req.Header.Set("Content-Type", "application/json")
	client := &http.Client{}
	resp, err := client.Do(req)
	assert.Equal(t, http.StatusBadRequest, resp.StatusCode, "Request should work")
}