Golang levigo.NewBloomFilter函数代码示例

func NewLevelDbShardDatastore(config *configuration.Configuration) (*LevelDbShardDatastore, error) {
	baseDbDir := filepath.Join(config.DataDir, SHARD_DATABASE_DIR)
	err := os.MkdirAll(baseDbDir, 0744)
	if err != nil {
		return nil, err
	}
	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(config.LevelDbLruCacheSize))
	opts.SetCreateIfMissing(true)
	opts.SetBlockSize(64 * ONE_KILOBYTE)
	filter := levigo.NewBloomFilter(SHARD_BLOOM_FILTER_BITS_PER_KEY)
	opts.SetFilterPolicy(filter)
	opts.SetMaxOpenFiles(config.LevelDbMaxOpenFiles)

	return &LevelDbShardDatastore{
		baseDbDir:      baseDbDir,
		config:         config,
		shards:         make(map[uint32]*LevelDbShard),
		levelDbOptions: opts,
		maxOpenShards:  config.LevelDbMaxOpenShards,
		lastAccess:     make(map[uint32]int64),
		shardRefCounts: make(map[uint32]int),
		shardsToClose:  make(map[uint32]bool),
		pointBatchSize: config.LevelDbPointBatchSize,
	}, nil
}

func NewLeveldbCache(dbname string, cacheM int) (*LeveldbCache, error) {
	opts := levigo.NewOptions()
	filter := levigo.NewBloomFilter(10)
	cache := levigo.NewLRUCache(1024 * 1024 * cacheM)
	opts.SetFilterPolicy(filter)
	opts.SetCache(cache)
	opts.SetCreateIfMissing(true)
	opts.SetWriteBufferSize(8 * 1024 * 104) // 8M
	opts.SetCompression(levigo.SnappyCompression)

	if ldb, err := levigo.Open(dbname, opts); err == nil {
		so := levigo.NewReadOptions()
		so.SetFillCache(false)
		return &LeveldbCache{
			db:    ldb,
			fp:    filter,
			cache: cache,
			Ro:    levigo.NewReadOptions(),
			Wo:    levigo.NewWriteOptions(),
			So:    so,
		}, nil
	} else {
		return nil, err
	}
}

func (db *DB) initOptions(cfg *Config) *levigo.Options {
	opts := levigo.NewOptions()

	opts.SetCreateIfMissing(true)

	if cfg.CacheSize > 0 {
		db.cache = levigo.NewLRUCache(cfg.CacheSize)
		opts.SetCache(db.cache)
	}

	//we must use bloomfilter
	db.filter = levigo.NewBloomFilter(defaultFilterBits)
	opts.SetFilterPolicy(db.filter)

	if !cfg.Compression {
		opts.SetCompression(levigo.NoCompression)
	}

	if cfg.BlockSize > 0 {
		opts.SetBlockSize(cfg.BlockSize)
	}

	if cfg.WriteBufferSize > 0 {
		opts.SetWriteBufferSize(cfg.WriteBufferSize)
	}

	return opts
}

func NewLevelDbDatastore(dbDir string) (Datastore, error) {
	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(ONE_GIGABYTE))
	opts.SetCreateIfMissing(true)
	opts.SetBlockSize(TWO_FIFTY_SIX_KILOBYTES)
	filter := levigo.NewBloomFilter(BLOOM_FILTER_BITS_PER_KEY)
	opts.SetFilterPolicy(filter)
	db, err := levigo.Open(dbDir, opts)
	if err != nil {
		return nil, err
	}

	ro := levigo.NewReadOptions()

	lastIdBytes, err2 := db.Get(ro, NEXT_ID_KEY)
	if err2 != nil {
		return nil, err2
	}

	lastId := uint64(0)
	if lastIdBytes != nil {
		lastId, err2 = binary.ReadUvarint(bytes.NewBuffer(lastIdBytes))
		if err2 != nil {
			return nil, err2
		}
	}

	wo := levigo.NewWriteOptions()

	return &LevelDbDatastore{db: db, lastIdUsed: lastId, readOptions: ro, writeOptions: wo}, nil
}

func NewLevelDBPersistence(storageRoot string, cacheCapacity, bitsPerBloomFilterEncoded int) (p *LevelDBPersistence, err error) {
	options := levigo.NewOptions()
	options.SetCreateIfMissing(true)
	options.SetParanoidChecks(true)

	cache := levigo.NewLRUCache(cacheCapacity)
	options.SetCache(cache)

	filterPolicy := levigo.NewBloomFilter(bitsPerBloomFilterEncoded)
	options.SetFilterPolicy(filterPolicy)

	storage, err := levigo.Open(storageRoot, options)
	if err != nil {
		return
	}

	readOptions := levigo.NewReadOptions()
	writeOptions := levigo.NewWriteOptions()
	writeOptions.SetSync(true)

	p = &LevelDBPersistence{
		cache:        cache,
		filterPolicy: filterPolicy,
		options:      options,
		readOptions:  readOptions,
		storage:      storage,
		writeOptions: writeOptions,
	}

	return
}

func (engine *LevelDbEngine) Init(config *proto.DBConfigs) error {
	if config == nil {
		return proto.ErrNoEngineConfig
	}
	if config.LevelDbConfigs == nil {
		config.LevelDbConfigs = DefaultLevelDbConf
	}
	options := levigo.NewOptions()
	// options.SetCreateIfMissing(config.CreateIfMissing)
	options.SetCreateIfMissing(true)
	options.SetParanoidChecks(config.LevelDbConfigs.ParanoidCheck)
	if config.LevelDbConfigs.LRUCacheSize > 0 {
		options.SetCache(levigo.NewLRUCache(config.LevelDbConfigs.LRUCacheSize))
	}
	if config.LevelDbConfigs.BloomFilterLength > 0 {
		options.SetFilterPolicy(levigo.NewBloomFilter(config.LevelDbConfigs.BloomFilterLength))
	}
	engine.config = config
	engine.dbOptions = options
	db, err := levigo.Open(engine.config.DataPath, engine.dbOptions)
	if err != nil {
		return err
	}
	engine.db = db
	return nil
}

func TestLevigo(t *testing.T) {
	path := "/tmp/levigo_test_10101"
	os.RemoveAll(path)

	opts := levigo.NewOptions()
	filter := levigo.NewBloomFilter(10)
	opts.SetFilterPolicy(filter)
	opts.SetCache(levigo.NewLRUCache(1024 << 20)) // 1G
	opts.SetCreateIfMissing(true)
	if ldb, err := levigo.Open(path, opts); err == nil {
		key := []byte("test-test hwl0dsfds")
		val := []byte("value")

		if err = ldb.Put(levigo.NewWriteOptions(), key, val); err != nil {
			t.Fail()
		} else {
			ro := levigo.NewReadOptions()
			if data, err := ldb.Get(ro, key); err == nil && reflect.DeepEqual(data, val) {
				ro.SetFillCache(false)
				it := ldb.NewIterator(ro)
				it.Seek([]byte{0})
				for ; it.Valid(); it.Next() {
					log.Printf("%s => %s", it.Key(), it.Value())
				}
			} else {
				t.Fail()
			}
		}
	} else {
		t.Fail()
	}
}

func NewLevelDbDatastore(dbDir string) (Datastore, error) {
	mainDbDir := filepath.Join(dbDir, DATABASE_DIR)
	requestLogDir := filepath.Join(dbDir, REQUEST_LOG_BASE_DIR)

	err := os.MkdirAll(mainDbDir, 0744)
	if err != nil {
		return nil, err
	}
	previousLog, err := NewRequestLogDb(getRequestLogDirForDate(requestLogDir, time.Now().Add(-time.Hour*24)))
	if err != nil {
		return nil, err
	}
	currentLog, err := NewRequestLogDb(getRequestLogDirForDate(requestLogDir, time.Now()))
	if err != nil {
		return nil, err
	}

	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(ONE_GIGABYTE))
	opts.SetCreateIfMissing(true)
	opts.SetBlockSize(TWO_FIFTY_SIX_KILOBYTES)
	filter := levigo.NewBloomFilter(BLOOM_FILTER_BITS_PER_KEY)
	opts.SetFilterPolicy(filter)
	db, err := levigo.Open(dbDir, opts)
	if err != nil {
		return nil, err
	}

	ro := levigo.NewReadOptions()

	lastIdBytes, err2 := db.Get(ro, NEXT_ID_KEY)
	if err2 != nil {
		return nil, err2
	}

	lastId := uint64(0)
	if lastIdBytes != nil {
		lastId, err2 = binary.ReadUvarint(bytes.NewBuffer(lastIdBytes))
		if err2 != nil {
			return nil, err2
		}
	}

	wo := levigo.NewWriteOptions()

	leveldbStore := &LevelDbDatastore{
		db:                 db,
		lastIdUsed:         lastId,
		readOptions:        ro,
		writeOptions:       wo,
		requestLogDir:      requestLogDir,
		currentRequestLog:  currentLog,
		previousRequestLog: previousLog}

	go leveldbStore.periodicallyRotateRequestLog()

	return leveldbStore, nil
}

func openDB() {
	opts := levigo.NewOptions()
	cache := levigo.NewLRUCache(128 * 1024 * 1024) // 128MB cache
	opts.SetCache(cache)
	filter := levigo.NewBloomFilter(10)
	opts.SetFilterPolicy(filter)
	opts.SetCreateIfMissing(true)

	var err error
	DB, err = levigo.Open("db", opts)
	maybeFatal(err)
}

func NewTorrentDB(dir string) (*TorrentDB, error) {
	opts := levigo.NewOptions()
	filter := levigo.NewBloomFilter(10)
	opts.SetFilterPolicy(filter)
	opts.SetCreateIfMissing(true)
	defer opts.Close()
	db, err := levigo.Open(dir, opts)
	if err != nil {
		return nil, err
	}
	return &TorrentDB{db, nil, &sync.RWMutex{}}, nil
}

func openDb(path string) (*levigo.DB, error) {
	opts := levigo.NewOptions()
	opts.SetCreateIfMissing(true)
	opts.SetFilterPolicy(levigo.NewBloomFilter(16))
	opts.SetCache(levigo.NewLRUCache(10490000))
	opts.SetMaxOpenFiles(500)
	opts.SetWriteBufferSize(62914560)
	opts.SetEnv(levigo.NewDefaultEnv())
	dbn, err := levigo.Open(path, opts)
	if err != nil {
		return nil, fmt.Errorf("failed to open db at %s: %v\n", path, err)
	}
	return dbn, nil
}

func NewRequestLogDb(dir string) (*requestLogDb, error) {
	err := os.MkdirAll(dir, 0744)
	if err != nil {
		return nil, err
	}
	opts := levigo.NewOptions()
	opts.SetCache(levigo.NewLRUCache(ONE_MEGABYTE))
	opts.SetCreateIfMissing(true)
	opts.SetBlockSize(TWO_FIFTY_SIX_KILOBYTES)
	filter := levigo.NewBloomFilter(BLOOM_FILTER_BITS_PER_KEY)
	opts.SetFilterPolicy(filter)
	db, err := levigo.Open(dir, opts)
	if err != nil {
		return nil, err
	}
	return &requestLogDb{dir: dir, db: db}, nil
}

// NewLevelDBPersistence returns an initialized LevelDBPersistence object,
// created with the given options.
func NewLevelDBPersistence(o LevelDBOptions) (*LevelDBPersistence, error) {
	options := levigo.NewOptions()
	options.SetCreateIfMissing(true)
	options.SetParanoidChecks(o.UseParanoidChecks)

	compression := levigo.SnappyCompression
	if o.Compression == Uncompressed {
		compression = levigo.NoCompression
	}
	options.SetCompression(compression)

	cache := levigo.NewLRUCache(o.CacheSizeBytes)
	options.SetCache(cache)

	filterPolicy := levigo.NewBloomFilter(10)
	options.SetFilterPolicy(filterPolicy)

	options.SetMaxOpenFiles(o.OpenFileAllowance)

	storage, err := levigo.Open(o.Path, options)
	if err != nil {
		return nil, err
	}

	readOptions := levigo.NewReadOptions()

	writeOptions := levigo.NewWriteOptions()
	writeOptions.SetSync(o.FlushOnMutate)

	return &LevelDBPersistence{
		path:    o.Path,
		name:    o.Name,
		purpose: o.Purpose,

		cache:        cache,
		filterPolicy: filterPolicy,

		options:      options,
		readOptions:  readOptions,
		writeOptions: writeOptions,

		storage: storage,
	}, nil
}

func applyConfig(o *levigo.Options, config map[string]interface{}) (
	*levigo.Options, error) {

	cim, ok := config["create_if_missing"].(bool)
	if ok {
		o.SetCreateIfMissing(cim)
	}

	eie, ok := config["error_if_exists"].(bool)
	if ok {
		o.SetErrorIfExists(eie)
	}

	wbs, ok := config["write_buffer_size"].(float64)
	if ok {
		o.SetWriteBufferSize(int(wbs))
	}

	bs, ok := config["block_size"].(float64)
	if ok {
		o.SetBlockSize(int(bs))
	}

	bri, ok := config["block_restart_interval"].(float64)
	if ok {
		o.SetBlockRestartInterval(int(bri))
	}

	lcc, ok := config["lru_cache_capacity"].(float64)
	if ok {
		lruCache := levigo.NewLRUCache(int(lcc))
		o.SetCache(lruCache)
	}

	bfbpk, ok := config["bloom_filter_bits_per_key"].(float64)
	if ok {
		bf := levigo.NewBloomFilter(int(bfbpk))
		o.SetFilterPolicy(bf)
	}

	return o, nil
}

func NewLevelDBPersistence(storageRoot string, cacheCapacity, bitsPerBloomFilterEncoded int) (p *LevelDBPersistence, err error) {
	options := levigo.NewOptions()
	options.SetCreateIfMissing(true)
	options.SetParanoidChecks(*leveldbUseParanoidChecks)
	compression := levigo.NoCompression
	if *leveldbUseSnappy {
		compression = levigo.SnappyCompression
	}
	options.SetCompression(compression)

	cache := levigo.NewLRUCache(cacheCapacity)
	options.SetCache(cache)

	filterPolicy := levigo.NewBloomFilter(bitsPerBloomFilterEncoded)
	options.SetFilterPolicy(filterPolicy)

	storage, err := levigo.Open(storageRoot, options)
	if err != nil {
		return
	}

	var (
		readOptions  = levigo.NewReadOptions()
		writeOptions = levigo.NewWriteOptions()
	)

	writeOptions.SetSync(*leveldbFlushOnMutate)
	p = &LevelDBPersistence{
		cache:        cache,
		filterPolicy: filterPolicy,
		options:      options,
		readOptions:  readOptions,
		storage:      storage,
		writeOptions: writeOptions,
	}

	return
}