Golang tsdb.NewStore函数代码示例

func TestStoreEnsureSeriesPersistedInNewShards(t *testing.T) {
	dir, err := ioutil.TempDir("", "store_test")
	if err != nil {
		t.Fatalf("Store.Open() failed to create temp dir: %v", err)
	}
	defer os.RemoveAll(dir)

	s := tsdb.NewStore(dir)
	s.EngineOptions.Config.WALDir = filepath.Join(dir, "wal")
	if err := s.Open(); err != nil {
		t.Fatalf("Store.Open() failed: %v", err)
	}

	if err := s.CreateShard("foo", "default", 1); err != nil {
		t.Fatalf("error creating shard: %v", err)
	}

	p, _ := tsdb.ParsePoints([]byte("cpu val=1"))
	if err := s.WriteToShard(1, p); err != nil {
		t.Fatalf("error writing to shard: %v", err)
	}

	if err := s.CreateShard("foo", "default", 2); err != nil {
		t.Fatalf("error creating shard: %v", err)
	}

	if err := s.WriteToShard(2, p); err != nil {
		t.Fatalf("error writing to shard: %v", err)
	}

	d := s.DatabaseIndex("foo")
	if d == nil {
		t.Fatal("expected to have database index for foo")
	}
	if d.Series("cpu") == nil {
		t.Fatal("expected series cpu to be in the index")
	}

	// delete the shard, close the store and reopen it and confirm the measurement is still there
	s.DeleteShard(1)
	s.Close()

	s = tsdb.NewStore(dir)
	s.EngineOptions.Config.WALDir = filepath.Join(dir, "wal")
	if err := s.Open(); err != nil {
		t.Fatalf("Store.Open() failed: %v", err)
	}

	d = s.DatabaseIndex("foo")
	if d == nil {
		t.Fatal("expected to have database index for foo")
	}
	if d.Series("cpu") == nil {
		t.Fatal("expected series cpu to be in the index")
	}
}

func TestDropMeasurementStatement(t *testing.T) {
	store, executor := testStoreAndExecutor()
	defer os.RemoveAll(store.Path())

	pt := tsdb.NewPoint(
		"cpu",
		map[string]string{"host": "server"},
		map[string]interface{}{"value": 1.0},
		time.Unix(1, 2),
	)
	pt2 := tsdb.NewPoint(
		"memory",
		map[string]string{"host": "server"},
		map[string]interface{}{"value": 1.0},
		time.Unix(1, 2),
	)

	if err := store.WriteToShard(shardID, []tsdb.Point{pt, pt2}); err != nil {
		t.Fatal(err)
	}

	got := executeAndGetJSON("show series", executor)
	exepected := `[{"series":[{"name":"cpu","columns":["_key","host"],"values":[["cpu,host=server","server"]]},{"name":"memory","columns":["_key","host"],"values":[["memory,host=server","server"]]}]}]`
	if exepected != got {
		t.Fatalf("exp: %s\ngot: %s", exepected, got)
	}

	got = executeAndGetJSON("drop measurement memory", executor)
	exepected = `[{}]`
	if exepected != got {
		t.Fatalf("exp: %s\ngot: %s", exepected, got)
	}

	validateDrop := func() {
		got = executeAndGetJSON("show series", executor)
		exepected = `[{"series":[{"name":"cpu","columns":["_key","host"],"values":[["cpu,host=server","server"]]}]}]`
		if exepected != got {
			t.Fatalf("exp: %s\ngot: %s", exepected, got)
		}
		got = executeAndGetJSON("show measurements", executor)
		exepected = `[{"series":[{"name":"measurements","columns":["name"],"values":[["cpu"]]}]}]`
		if exepected != got {
			t.Fatalf("exp: %s\ngot: %s", exepected, got)
		}
		got = executeAndGetJSON("select * from memory", executor)
		exepected = `[{}]`
		if exepected != got {
			t.Fatalf("exp: %s\ngot: %s", exepected, got)
		}
	}

	validateDrop()
	store.Close()
	conf := store.EngineOptions.Config
	store = tsdb.NewStore(store.Path())
	store.EngineOptions.Config = conf
	store.Open()
	executor.Store = store
	validateDrop()
}

func TestStoreOpenNotRPDir(t *testing.T) {
	dir, err := ioutil.TempDir("", "store_test")
	if err != nil {
		t.Fatalf("Store.Open() failed to create temp dir: %v", err)
	}

	path := filepath.Join(dir, "mydb")
	if err := os.MkdirAll(path, 0700); err != nil {
		t.Fatalf("Store.Open() failed to create test db dir: %v", err)
	}

	rpPath := filepath.Join(path, "myrp")
	if _, err := os.Create(rpPath); err != nil {
		t.Fatalf("Store.Open() failed to create test retention policy directory: %v", err)
	}

	s := tsdb.NewStore(dir)
	s.EngineOptions.Config.WALDir = filepath.Join(dir, "wal")
	if err := s.Open(); err != nil {
		t.Fatalf("Store.Open() failed: %v", err)
	}

	if got, exp := s.DatabaseIndexN(), 1; got != exp {
		t.Fatalf("Store.Open() database index count mismatch: got %v, exp %v", got, exp)
	}

	if di := s.DatabaseIndex("mydb"); di == nil {
		t.Errorf("Store.Open() database myb does not exist")
	}

	if got, exp := s.ShardN(), 0; got != exp {
		t.Fatalf("Store.Open() shard count mismatch: got %v, exp %v", got, exp)
	}
}

func TestDropDatabase(t *testing.T) {
	store, executor := testStoreAndExecutor("")
	defer os.RemoveAll(store.Path())

	pt := models.NewPoint(
		"cpu",
		map[string]string{"host": "server"},
		map[string]interface{}{"value": 1.0},
		time.Unix(1, 2),
	)

	if err := store.WriteToShard(shardID, []models.Point{pt}); err != nil {
		t.Fatal(err)
	}

	got := executeAndGetJSON("SELECT * FROM cpu GROUP BY *", executor)
	expected := `[{"series":[{"name":"cpu","tags":{"host":"server"},"columns":["time","value"],"values":[["1970-01-01T00:00:01.000000002Z",1]]}]}]`
	if expected != got {
		t.Fatalf("exp: %s\ngot: %s", expected, got)
	}

	var name string
	me := &metaExec{fn: func(stmt influxql.Statement) *influxql.Result {
		name = stmt.(*influxql.DropDatabaseStatement).Name
		return &influxql.Result{}
	}}
	executor.MetaStatementExecutor = me

	// verify the database is there on disk
	dbPath := filepath.Join(store.Path(), "foo")
	if _, err := os.Stat(dbPath); err != nil {
		t.Fatalf("execpted database dir %s to exist", dbPath)
	}

	got = executeAndGetJSON("drop database foo", executor)
	expected = `[{}]`
	if got != expected {
		t.Fatalf("exp: %s\ngot: %s", expected, got)
	}

	if name != "foo" {
		t.Fatalf("expected the MetaStatementExecutor to be called with database name foo, but got %s", name)
	}

	if _, err := os.Stat(dbPath); !os.IsNotExist(err) {
		t.Fatalf("expected database dir %s to be gone", dbPath)
	}

	store.Close()
	conf := store.EngineOptions.Config
	store = tsdb.NewStore(store.Path())
	store.EngineOptions.Config = conf
	store.Open()
	executor.Store = store
	executor.ShardMapper = &testShardMapper{store: store}

	if err := store.WriteToShard(shardID, []models.Point{pt}); err == nil || err.Error() != "shard not found" {
		t.Fatalf("expected shard to not be found")
	}
}

func TestStoreOpenNotDatabaseDir(t *testing.T) {
	dir, err := ioutil.TempDir("", "store_test")
	if err != nil {
		t.Fatalf("Store.Open() failed to create temp dir: %v", err)
	}
	defer os.RemoveAll(dir)

	path := filepath.Join(dir, "bad_db_path")
	if _, err := os.Create(path); err != nil {
		t.Fatalf("Store.Open() failed to create test db dir: %v", err)
	}

	s := tsdb.NewStore(dir)
	s.EngineOptions.Config.WALDir = filepath.Join(dir, "wal")
	if err := s.Open(); err != nil {
		t.Fatalf("Store.Open() failed: %v", err)
	}

	if got, exp := s.DatabaseIndexN(), 0; got != exp {
		t.Fatalf("Store.Open() database index count mismatch: got %v, exp %v", got, exp)
	}

	if got, exp := s.ShardN(), 0; got != exp {
		t.Fatalf("Store.Open() shard count mismatch: got %v, exp %v", got, exp)
	}
}

// NewServer returns a new instance of Server built from a config.
func NewServer(c *Config, version string) (*Server, error) {
	// Construct base meta store and data store.
	s := &Server{
		version: version,
		err:     make(chan error),
		closing: make(chan struct{}),

		Hostname:    c.Meta.Hostname,
		BindAddress: c.Meta.BindAddress,

		MetaStore: meta.NewStore(c.Meta),
		TSDBStore: tsdb.NewStore(c.Data.Dir),

		reportingDisabled: c.ReportingDisabled,
	}

	// Copy TSDB configuration.
	s.TSDBStore.MaxWALSize = c.Data.MaxWALSize
	s.TSDBStore.WALFlushInterval = time.Duration(c.Data.WALFlushInterval)

	// Initialize query executor.
	s.QueryExecutor = tsdb.NewQueryExecutor(s.TSDBStore)
	s.QueryExecutor.MetaStore = s.MetaStore
	s.QueryExecutor.MetaStatementExecutor = &meta.StatementExecutor{Store: s.MetaStore}

	// Set the shard writer
	s.ShardWriter = cluster.NewShardWriter(time.Duration(c.Cluster.ShardWriterTimeout))
	s.ShardWriter.MetaStore = s.MetaStore

	// Create the hinted handoff service
	s.HintedHandoff = hh.NewService(c.HintedHandoff, s.ShardWriter)

	// Initialize points writer.
	s.PointsWriter = cluster.NewPointsWriter()
	s.PointsWriter.MetaStore = s.MetaStore
	s.PointsWriter.TSDBStore = s.TSDBStore
	s.PointsWriter.ShardWriter = s.ShardWriter
	s.PointsWriter.HintedHandoff = s.HintedHandoff

	// Append services.
	s.appendClusterService(c.Cluster)
	s.appendPrecreatorService(c.Precreator)
	s.appendSnapshotterService()
	s.appendAdminService(c.Admin)
	s.appendContinuousQueryService(c.ContinuousQuery)
	s.appendHTTPDService(c.HTTPD)
	s.appendCollectdService(c.Collectd)
	if err := s.appendOpenTSDBService(c.OpenTSDB); err != nil {
		return nil, err
	}
	s.appendUDPService(c.UDP)
	s.appendRetentionPolicyService(c.Retention)
	for _, g := range c.Graphites {
		if err := s.appendGraphiteService(g); err != nil {
			return nil, err
		}
	}

	return s, nil
}

func benchmarkStoreOpen(b *testing.B, mCnt, tkCnt, tvCnt, pntCnt, shardCnt int) {
	// Generate test series (measurements + unique tag sets).
	series := genTestSeries(mCnt, tkCnt, tvCnt)
	// Generate point data to write to the shards.
	points := []tsdb.Point{}
	for _, s := range series {
		for val := 0.0; val < float64(pntCnt); val++ {
			p := tsdb.NewPoint(s.Measurement, s.Series.Tags, map[string]interface{}{"value": val}, time.Now())
			points = append(points, p)
		}
	}
	// Create a temporary directory for the test data.
	dir, _ := ioutil.TempDir("", "store_test")
	// Create the store.
	store := tsdb.NewStore(dir)
	// Open the store.
	if err := store.Open(); err != nil {
		b.Fatalf("benchmarkStoreOpen: %s", err)
	}
	// Create requested number of shards in the store & write points.
	for shardID := 0; shardID < shardCnt; shardID++ {
		if err := store.CreateShard("mydb", "myrp", uint64(shardID)); err != nil {
			b.Fatalf("benchmarkStoreOpen: %s", err)
		}
		// Write points to the shard.
		chunkedWriteStoreShard(store, shardID, points)
	}
	// Close the store.
	if err := store.Close(); err != nil {
		b.Fatalf("benchmarkStoreOpen: %s", err)
	}

	// Run the benchmark loop.
	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		store := tsdb.NewStore(dir)
		if err := store.Open(); err != nil {
			b.Fatalf("benchmarkStoreOpen: %s", err)
		}

		b.StopTimer()
		store.Close()
		b.StartTimer()
	}
}

func testStore() *tsdb.Store {
	path, _ := ioutil.TempDir("", "")

	store := tsdb.NewStore(path)
	err := store.Open()
	if err != nil {
		panic(err)
	}
	return store
}

func TestDropSeriesStatement(t *testing.T) {
	store, executor := testStoreAndExecutor("")
	defer os.RemoveAll(store.Path())

	pt := models.NewPoint(
		"cpu",
		map[string]string{"host": "server"},
		map[string]interface{}{"value": 1.0},
		time.Unix(1, 2),
	)

	err := store.WriteToShard(shardID, []models.Point{pt})
	if err != nil {
		t.Fatalf(err.Error())
	}

	got := executeAndGetJSON("SELECT * FROM cpu GROUP BY *", executor)
	exepected := `[{"series":[{"name":"cpu","tags":{"host":"server"},"columns":["time","value"],"values":[["1970-01-01T00:00:01.000000002Z",1]]}]}]`
	if exepected != got {
		t.Fatalf("exp: %s\ngot: %s", exepected, got)
	}

	got = executeAndGetJSON("drop series from cpu", executor)

	got = executeAndGetJSON("SELECT * FROM cpu GROUP BY *", executor)
	exepected = `[{}]`
	if exepected != got {
		t.Fatalf("exp: %s\ngot: %s", exepected, got)
	}

	got = executeAndGetJSON("show tag keys from cpu", executor)
	exepected = `[{}]`
	if exepected != got {
		t.Fatalf("exp: %s\ngot: %s", exepected, got)
	}

	store.Close()
	conf := store.EngineOptions.Config
	store = tsdb.NewStore(store.Path())
	store.EngineOptions.Config = conf
	store.Open()
	executor.Store = store

	got = executeAndGetJSON("select * from cpu", executor)
	exepected = `[{}]`
	if exepected != got {
		t.Fatalf("exp: %s\ngot: %s", exepected, got)
	}

	got = executeAndGetJSON("show tag keys from cpu", executor)
	exepected = `[{}]`
	if exepected != got {
		t.Fatalf("exp: %s\ngot: %s", exepected, got)
	}
}

func testStore() *tsdb.Store {
	path, _ := ioutil.TempDir("", "")

	store := tsdb.NewStore(path)

	store.EngineOptions.Config.WALDir = filepath.Join(path, "wal")
	err := store.Open()
	if err != nil {
		panic(err)
	}
	return store
}

func TestWritePointsAndExecuteQuery(t *testing.T) {
	store, executor := testStoreAndExecutor("")
	defer os.RemoveAll(store.Path())

	// Write first point.
	if err := store.WriteToShard(shardID, []models.Point{models.NewPoint(
		"cpu",
		map[string]string{"host": "server"},
		map[string]interface{}{"value": 1.0},
		time.Unix(1, 2),
	)}); err != nil {
		t.Fatalf(err.Error())
	}

	// Write second point.
	if err := store.WriteToShard(shardID, []models.Point{models.NewPoint(
		"cpu",
		map[string]string{"host": "server"},
		map[string]interface{}{"value": 1.0},
		time.Unix(2, 3),
	)}); err != nil {
		t.Fatalf(err.Error())
	}

	got := executeAndGetJSON("SELECT * FROM cpu", executor)
	exepected := `[{"series":[{"name":"cpu","columns":["time","host","value"],"values":[["1970-01-01T00:00:01.000000002Z","server",1],["1970-01-01T00:00:02.000000003Z","server",1]]}]}]`
	if exepected != got {
		t.Fatalf("\nexp: %s\ngot: %s", exepected, got)
	}

	got = executeAndGetJSON("SELECT * FROM cpu GROUP BY *", executor)
	exepected = `[{"series":[{"name":"cpu","tags":{"host":"server"},"columns":["time","value"],"values":[["1970-01-01T00:00:01.000000002Z",1],["1970-01-01T00:00:02.000000003Z",1]]}]}]`
	if exepected != got {
		t.Fatalf("\nexp: %s\ngot: %s", exepected, got)
	}

	store.Close()
	conf := store.EngineOptions.Config
	store = tsdb.NewStore(store.Path())
	store.EngineOptions.Config = conf
	if err := store.Open(); err != nil {
		t.Fatalf(err.Error())
	}
	executor.Store = store
	executor.ShardMapper = &testShardMapper{store: store}

	got = executeAndGetJSON("SELECT * FROM cpu GROUP BY *", executor)
	if exepected != got {
		t.Fatalf("\nexp: %s\ngot: %s", exepected, got)
	}
}

func TestStoreOpenShardCreateDelete(t *testing.T) {
	dir, err := ioutil.TempDir("", "store_test")
	if err != nil {
		t.Fatalf("Store.Open() failed to create temp dir: %v", err)
	}

	path := filepath.Join(dir, "mydb", "myrp")
	if err := os.MkdirAll(path, 0700); err != nil {
		t.Fatalf("Store.Open() failed to create test db dir: %v", err)
	}

	s := tsdb.NewStore(dir)
	s.EngineOptions.Config.WALDir = filepath.Join(dir, "wal")
	if err := s.Open(); err != nil {
		t.Fatalf("Store.Open() failed: %v", err)
	}

	if got, exp := s.DatabaseIndexN(), 1; got != exp {
		t.Fatalf("Store.Open() database index count mismatch: got %v, exp %v", got, exp)
	}

	if di := s.DatabaseIndex("mydb"); di == nil {
		t.Errorf("Store.Open() database mydb does not exist")
	}

	if err := s.CreateShard("mydb", "myrp", 1); err != nil {
		t.Fatalf("Store.Open() failed to create shard")
	}

	if got, exp := s.ShardN(), 1; got != exp {
		t.Fatalf("Store.Open() shard count mismatch: got %v, exp %v", got, exp)
	}

	shardIDs := s.ShardIDs()
	if len(shardIDs) != 1 || shardIDs[0] != 1 {
		t.Fatalf("Store.Open() ShardIDs not correct: got %v, exp %v", s.ShardIDs(), []uint64{1})
	}

	if err := s.DeleteShard(1); err != nil {
		t.Fatalf("Store.Open() failed to delete shard: %v", err)
	}

	if sh := s.Shard(1); sh != nil {
		t.Fatal("Store.Open() shard ID 1 still exists")
	}
}

func testStoreAndExecutor() (*tsdb.Store, *tsdb.QueryExecutor) {
	path, _ := ioutil.TempDir("", "")

	store := tsdb.NewStore(path)
	err := store.Open()
	if err != nil {
		panic(err)
	}
	database := "foo"
	retentionPolicy := "bar"
	shardID := uint64(1)
	store.CreateShard(database, retentionPolicy, shardID)

	executor := tsdb.NewQueryExecutor(store)
	executor.MetaStore = &testMetastore{}
	executor.ShardMapper = &testShardMapper{store: store}

	return store, executor
}

func TestStoreOpen(t *testing.T) {
	dir, err := ioutil.TempDir("", "store_test")
	if err != nil {
		t.Fatalf("failed to create temp dir: %v", err)
	}
	defer os.RemoveAll(dir)

	if err := os.MkdirAll(filepath.Join(dir, "mydb"), 0600); err != nil {
		t.Fatalf("failed to create test db dir: %v", err)
	}

	s := tsdb.NewStore(dir)
	if err := s.Open(); err != nil {
		t.Fatalf("Store.Open() failed: %v", err)
	}

	if got, exp := s.DatabaseIndexN(), 1; got != exp {
		t.Fatalf("database index count mismatch: got %v, exp %v", got, exp)
	}
}

// Ensure writing a point and updating it results in only a single point.
func TestWritePointsAndExecuteQuery_Update(t *testing.T) {
	store, executor := testStoreAndExecutor("")
	defer os.RemoveAll(store.Path())

	// Write original point.
	if err := store.WriteToShard(1, []models.Point{models.NewPoint(
		"temperature",
		map[string]string{},
		map[string]interface{}{"value": 100.0},
		time.Unix(0, 0),
	)}); err != nil {
		t.Fatalf(err.Error())
	}

	// Restart store.
	store.Close()
	conf := store.EngineOptions.Config
	store = tsdb.NewStore(store.Path())
	store.EngineOptions.Config = conf
	if err := store.Open(); err != nil {
		t.Fatalf(err.Error())
	}
	executor.Store = store
	executor.ShardMapper = &testShardMapper{store: store}

	// Rewrite point with new value.
	if err := store.WriteToShard(1, []models.Point{models.NewPoint(
		"temperature",
		map[string]string{},
		map[string]interface{}{"value": 200.0},
		time.Unix(0, 0),
	)}); err != nil {
		t.Fatalf(err.Error())
	}

	got := executeAndGetJSON("select * from temperature", executor)
	exp := `[{"series":[{"name":"temperature","columns":["time","value"],"values":[["1970-01-01T00:00:00Z",200]]}]}]`
	if exp != got {
		t.Fatalf("\n\nexp: %s\ngot: %s", exp, got)
	}
}