Golang engine.NewRocksDB函数代码示例

func openStore(cmd *cobra.Command, dir string, stopper *stop.Stopper) (engine.Engine, error) {
	db := engine.NewRocksDB(roachpb.Attributes{}, dir, 512<<20, 10<<20, 0, stopper)
	if err := db.Open(); err != nil {
		return nil, err
	}
	return db, nil
}

// initEngine parses the store attributes as a colon-separated list
// and instantiates an engine based on the dir parameter. If dir parses
// to an integer, it's taken to mean an in-memory engine; otherwise,
// dir is treated as a path and a RocksDB engine is created.
func (ctx *Context) initEngine(attrsStr, path string, stopper *stop.Stopper) (engine.Engine, error) {
	attrs := parseAttributes(attrsStr)
	if size, err := strconv.ParseUint(path, 10, 64); err == nil {
		if size == 0 {
			return nil, errUnsizedInMemStore
		}
		return engine.NewInMem(attrs, int64(size), stopper), nil
	}
	return engine.NewRocksDB(attrs, path, ctx.CacheSize, stopper), nil
}

func openStore(cmd *cobra.Command, dir string, stopper *stop.Stopper) (engine.Engine, error) {
	initCacheSize()

	db := engine.NewRocksDB(roachpb.Attributes{}, dir,
		cliContext.CacheSize, cliContext.MemtableBudget, 0, stopper)
	if err := db.Open(); err != nil {
		return nil, err
	}
	return db, nil
}

// initEngine parses the store attributes as a colon-separated list
// and instantiates an engine based on the dir parameter. If dir parses
// to an integer, it's taken to mean an in-memory engine; otherwise,
// dir is treated as a path and a RocksDB engine is created.
func (ctx *Context) initEngine(attrsStr, path string) (engine.Engine, error) {
	attrs := parseAttributes(attrsStr)
	if size, err := strconv.ParseUint(path, 10, 64); err == nil {
		if size == 0 {
			return nil, util.Errorf("unable to initialize an in-memory store with capacity 0")
		}
		return engine.NewInMem(attrs, int64(size)), nil
		// TODO(spencer): should be using rocksdb for in-memory stores and
		// relegate the InMem engine to usage only from unittests.
	}
	return engine.NewRocksDB(attrs, path, ctx.CacheSize), nil
}

func openStore(cmd *cobra.Command, args []string, stopper *stop.Stopper) (engine.Engine, error) {
	if len(args) != 1 {
		return nil, errors.New("one argument is required")
	}
	dir := args[0]

	db := engine.NewRocksDB(roachpb.Attributes{}, dir, 0, 0, stopper)
	if err := db.Open(); err != nil {
		return nil, err
	}
	return db, nil
}

// initEngine parses the store attributes as a colon-separated list
// and instantiates an engine based on the dir parameter. If dir parses
// to an integer, it's taken to mean an in-memory engine; otherwise,
// dir is treated as a path and a RocksDB engine is created.
func (ctx *Context) initEngine(attrsStr, path string, stopper *stop.Stopper) (engine.Engine, error) {
	attrs := parseAttributes(attrsStr)
	if size, err := strconv.ParseUint(path, 10, 64); err == nil {
		if size == 0 {
			return nil, errUnsizedInMemStore
		}
		return engine.NewInMem(attrs, int64(size), stopper), nil
	}
	// TODO(peter): The comments and docs say that CacheSize and MemtableBudget
	// are split evenly if there are multiple stores, but we aren't doing that
	// currently.
	return engine.NewRocksDB(attrs, path, ctx.CacheSize, ctx.MemtableBudget, stopper), nil
}

// InitStores initializes ctx.Engines based on ctx.Stores.
func (ctx *Context) InitStores(stopper *stop.Stopper) error {
	// TODO(peter): The comments and docs say that CacheSize and MemtableBudget
	// are split evenly if there are multiple stores, but we aren't doing that
	// currently.
	for _, spec := range ctx.Stores.Specs {
		var sizeInBytes = spec.SizeInBytes
		if spec.InMemory {
			if spec.SizePercent > 0 {
				sysMem, err := GetTotalMemory()
				if err != nil {
					return fmt.Errorf("could not retrieve system memory")
				}
				sizeInBytes = int64(float64(sysMem) * spec.SizePercent / 100)
			}
			if sizeInBytes != 0 && sizeInBytes < minimumStoreSize {
				return fmt.Errorf("%f%% of memory is only %s bytes, which is below the minimum requirement of %s",
					spec.SizePercent, humanize.IBytes(uint64(sizeInBytes)), humanize.IBytes(uint64(minimumStoreSize)))
			}
			ctx.Engines = append(ctx.Engines, engine.NewInMem(spec.Attributes, uint64(sizeInBytes), stopper))
		} else {
			if spec.SizePercent > 0 {
				fileSystemUsage := gosigar.FileSystemUsage{}
				if err := fileSystemUsage.Get(spec.Path); err != nil {
					return err
				}
				sizeInBytes = int64(float64(fileSystemUsage.Total) * spec.SizePercent / 100)
			}
			if sizeInBytes != 0 && sizeInBytes < minimumStoreSize {
				return fmt.Errorf("%f%% of %s's total free space is only %s bytes, which is below the minimum requirement of %s",
					spec.SizePercent, spec.Path, humanize.IBytes(uint64(sizeInBytes)),
					humanize.IBytes(uint64(minimumStoreSize)))
			}
			ctx.Engines = append(ctx.Engines, engine.NewRocksDB(spec.Attributes, spec.Path, ctx.CacheSize,
				ctx.MemtableBudget, sizeInBytes, stopper))
		}
	}
	if len(ctx.Engines) == 1 {
		log.Infof("1 storage engine initialized")
	} else {
		log.Infof("%d storage engines initialized", len(ctx.Engines))
	}
	return nil
}

// TestResponseCacheGC verifies that response cache entries are
// garbage collected periodically.
func TestResponseCacheGC(t *testing.T) {
	loc := util.CreateTempDirectory()
	rocksdb := engine.NewRocksDB(proto.Attributes{Attrs: []string{"ssd"}}, loc)
	if err := rocksdb.Start(); err != nil {
		t.Fatalf("could not create new rocksdb db instance at %s: %v", loc, err)
	}
	defer func(t *testing.T) {
		rocksdb.Close()
		if err := rocksdb.Destroy(); err != nil {
			t.Errorf("could not destroy rocksdb db at %s: %v", loc, err)
		}
	}(t)

	rc := NewResponseCache(1, rocksdb)
	cmdID := makeCmdID(1, 1)

	// Add response for cmdID with timestamp at time=1ns.
	copyIncR := incR
	copyIncR.Timestamp.WallTime = 1
	if err := rc.PutResponse(cmdID, &copyIncR); err != nil {
		t.Fatalf("unexpected error putting responpse: %v", err)
	}
	rocksdb.SetGCTimeouts(func() (minTxnTS, minRCacheTS int64) {
		minRCacheTS = 0 // avoids GC
		return
	})
	rocksdb.CompactRange(nil, nil)
	val := proto.IncrementResponse{}
	if ok, err := rc.GetResponse(cmdID, &val); !ok || err != nil || val.NewValue != 1 {
		t.Fatalf("unexpected response or error: %t, %v, %+v", ok, err, val)
	}

	// Now set minRCacheTS to 1, which will GC.
	rocksdb.SetGCTimeouts(func() (minTxnTS, minRCacheTS int64) {
		minRCacheTS = 1
		return
	})
	rocksdb.CompactRange(nil, nil)
	if ok, err := rc.GetResponse(cmdID, &val); ok || err != nil {
		t.Errorf("unexpected response or error: %t, %v", ok, err)
	}
}

// InitStores initializes ctx.Engines based on ctx.Stores.
func (ctx *Context) InitStores(stopper *stop.Stopper) error {
	cache := engine.NewRocksDBCache(ctx.CacheSize)
	defer cache.Release()

	for _, spec := range ctx.Stores.Specs {
		var sizeInBytes = spec.SizeInBytes
		if spec.InMemory {
			if spec.SizePercent > 0 {
				sysMem, err := GetTotalMemory()
				if err != nil {
					return fmt.Errorf("could not retrieve system memory")
				}
				sizeInBytes = int64(float64(sysMem) * spec.SizePercent / 100)
			}
			if sizeInBytes != 0 && sizeInBytes < minimumStoreSize {
				return fmt.Errorf("%f%% of memory is only %s bytes, which is below the minimum requirement of %s",
					spec.SizePercent, humanizeutil.IBytes(sizeInBytes), humanizeutil.IBytes(minimumStoreSize))
			}
			ctx.Engines = append(ctx.Engines, engine.NewInMem(spec.Attributes, sizeInBytes, stopper))
		} else {
			if spec.SizePercent > 0 {
				fileSystemUsage := gosigar.FileSystemUsage{}
				if err := fileSystemUsage.Get(spec.Path); err != nil {
					return err
				}
				sizeInBytes = int64(float64(fileSystemUsage.Total) * spec.SizePercent / 100)
			}
			if sizeInBytes != 0 && sizeInBytes < minimumStoreSize {
				return fmt.Errorf("%f%% of %s's total free space is only %s bytes, which is below the minimum requirement of %s",
					spec.SizePercent, spec.Path, humanizeutil.IBytes(sizeInBytes), humanizeutil.IBytes(minimumStoreSize))
			}
			ctx.Engines = append(ctx.Engines, engine.NewRocksDB(spec.Attributes, spec.Path,
				cache, ctx.MemtableBudget, sizeInBytes, stopper))
		}
	}
	if len(ctx.Engines) == 1 {
		log.Infof("1 storage engine initialized")
	} else {
		log.Infof("%d storage engines initialized", len(ctx.Engines))
	}
	return nil
}

func openStore(cmd *cobra.Command, dir string, stopper *stop.Stopper) (*engine.RocksDB, error) {
	cache := engine.NewRocksDBCache(512 << 20)
	defer cache.Release()
	maxOpenFiles, err := server.SetOpenFileLimitForOneStore()
	if err != nil {
		return nil, err
	}
	db := engine.NewRocksDB(
		roachpb.Attributes{},
		dir,
		cache,
		10<<20,
		0,
		maxOpenFiles,
		stopper,
	)
	if err := db.Open(); err != nil {
		return nil, err
	}
	return db, nil
}

func setupClientBenchData(useSSL bool, numVersions, numKeys int, b *testing.B) (
	*server.TestServer, *client.DB) {
	const cacheSize = 8 << 30        // 8 GB
	const memtableBudget = 512 << 20 // 512 MB
	loc := fmt.Sprintf("client_bench_%d_%d", numVersions, numKeys)

	exists := true
	if _, err := os.Stat(loc); os.IsNotExist(err) {
		exists = false
	}

	s := &server.TestServer{}
	s.Ctx = server.NewTestContext()
	s.SkipBootstrap = exists
	if !useSSL {
		s.Ctx.Insecure = true
	}
	stopper := stop.NewStopper()
	s.Ctx.Engines = []engine.Engine{
		engine.NewRocksDB(roachpb.Attributes{Attrs: []string{"ssd"}}, loc,
			cacheSize, memtableBudget, stopper),
	}
	if err := s.StartWithStopper(stopper); err != nil {
		b.Fatal(err)
	}

	db, err := client.Open(s.Stopper(), fmt.Sprintf("%s://%[email protected]%s?certs=%s",
		s.Ctx.RPCRequestScheme(), security.NodeUser, s.ServingAddr(), s.Ctx.Certs))
	if err != nil {
		b.Fatal(err)
	}

	if exists {
		return s, db
	}

	rng, _ := randutil.NewPseudoRand()
	keys := make([]roachpb.Key, numKeys)
	nvs := make([]int, numKeys)
	for t := 1; t <= numVersions; t++ {
		batch := &client.Batch{}
		for i := 0; i < numKeys; i++ {
			if t == 1 {
				keys[i] = roachpb.Key(encoding.EncodeUvarintAscending([]byte("key-"), uint64(i)))
				nvs[i] = int(rand.Int31n(int32(numVersions)) + 1)
			}
			// Only write values if this iteration is less than the random
			// number of versions chosen for this key.
			if t <= nvs[i] {
				batch.Put(roachpb.Key(keys[i]), randutil.RandBytes(rng, valueSize))
			}
			if (i+1)%1000 == 0 {
				if pErr := db.Run(batch); pErr != nil {
					b.Fatal(pErr)
				}
				batch = &client.Batch{}
			}
		}
		if len(batch.Results) != 0 {
			if pErr := db.Run(batch); pErr != nil {
				b.Fatal(pErr)
			}
		}
	}

	if r, ok := s.Ctx.Engines[0].(*engine.RocksDB); ok {
		r.CompactRange(engine.NilKey, engine.NilKey)
	}

	return s, db
}

func setupClientBenchData(useRPC, useSSL bool, numVersions, numKeys int, b *testing.B) (
	*server.TestServer, *client.DB) {
	const cacheSize = 8 << 30 // 8 GB
	loc := fmt.Sprintf("client_bench_%d_%d", numVersions, numKeys)

	exists := true
	if _, err := os.Stat(loc); os.IsNotExist(err) {
		exists = false
	}

	s := &server.TestServer{}
	s.Ctx = server.NewTestContext()
	s.Ctx.ExperimentalRPCServer = true
	s.SkipBootstrap = exists
	if !useSSL {
		s.Ctx.Insecure = true
	}
	s.Engines = []engine.Engine{engine.NewRocksDB(proto.Attributes{Attrs: []string{"ssd"}}, loc, cacheSize)}
	if err := s.Start(); err != nil {
		b.Fatal(err)
	}

	var scheme string
	if useRPC {
		scheme = "rpcs"
	} else {
		scheme = "https"
	}

	db, err := client.Open(scheme + "://[email protected]" + s.ServingAddr() + "?certs=" + s.Ctx.Certs)
	if err != nil {
		b.Fatal(err)
	}

	if exists {
		return s, db
	}

	rng, _ := util.NewPseudoRand()
	keys := make([]proto.Key, numKeys)
	nvs := make([]int, numKeys)
	for t := 1; t <= numVersions; t++ {
		batch := &client.Batch{}
		for i := 0; i < numKeys; i++ {
			if t == 1 {
				keys[i] = proto.Key(encoding.EncodeUvarint([]byte("key-"), uint64(i)))
				nvs[i] = int(rand.Int31n(int32(numVersions)) + 1)
			}
			// Only write values if this iteration is less than the random
			// number of versions chosen for this key.
			if t <= nvs[i] {
				batch.Put(proto.Key(keys[i]), util.RandBytes(rng, valueSize))
			}
			if (i+1)%1000 == 0 {
				if err := db.Run(batch); err != nil {
					b.Fatal(err)
				}
				batch = &client.Batch{}
			}
		}
		if len(batch.Results) != 0 {
			if err := db.Run(batch); err != nil {
				b.Fatal(err)
			}
		}
	}

	if r, ok := s.Engines[0].(*engine.RocksDB); ok {
		r.CompactRange(nil, nil)
	}

	return s, db
}