Golang chainstore.Store类代码示例

func TestFileStore(t *testing.T) {
	var store chainstore.Store
	var err error

	ctx := context.Background()

	store = chainstore.New(New(tempDir(), 0755))

	assert := assert.New(t)

	err = store.Open()
	assert.Nil(err)

	// Put/Get/Del basic data
	err = store.Put(ctx, "test.txt", []byte{1, 2, 3, 4})
	assert.Nil(err)

	data, err := store.Get(ctx, "test.txt")
	assert.Nil(err)
	assert.Equal(data, []byte{1, 2, 3, 4})

	// Auto-creating directories on put
	err = store.Put(ctx, "hello/there/everyone.txt", []byte{1, 2, 3, 4})
	assert.Nil(err)
}

func (cf *Config) GetChainstore() (chainstore.Store, error) {
	// First, reset cache storage path
	err := filepath.Walk(
		cf.Chainstore.Path,
		func(path string, info os.FileInfo, err error) error {
			if cf.Chainstore.Path == path {
				return nil // skip the root
			}
			if err = os.RemoveAll(path); err != nil {
				return fmt.Errorf("Failed to remove or clean the directory: %s, because: %s", path, err)
			}
			return nil
		},
	)
	if err != nil {
		return nil, err
	}

	// TODO: impl another kind of lrumgr (or option) to be based on number of keys, not filesize
	// at which point, we can add a method called .Keys() that will return the keys
	// matching some query from a Store, and we can seed the LRU this way, and keep
	// the bolt data..

	// Build the stores and setup the chain
	memStore := memstore.New(cf.Chainstore.MemCacheSize * 1024 * 1024)

	diskStore := lrumgr.New(cf.Chainstore.DiskCacheSize*1024*1024,
		metricsmgr.New("fn.store.bolt", nil,
			levelstore.New(cf.Chainstore.Path),
		),
	)

	var store chainstore.Store

	if cf.Chainstore.S3AccessKey != "" && cf.Chainstore.S3SecretKey != "" {
		s3Store := metricsmgr.New("fn.store.s3", nil,
			s3store.New(cf.Chainstore.S3Bucket, cf.Chainstore.S3AccessKey, cf.Chainstore.S3SecretKey),
		)

		store = chainstore.New(memStore, chainstore.Async(diskStore, s3Store))
	} else {
		store = chainstore.New(memStore, chainstore.Async(diskStore))
	}

	if err := store.Open(); err != nil {
		return nil, err
	}
	return store, nil
}

func (cf *Config) GetChainstore() (chainstore.Store, error) {
	// chainstore.DefaultTimeout = 60 * time.Second // TODO: ....

	// First, reset cache storage path
	err := filepath.Walk(
		cf.Chainstore.Path,
		func(path string, info os.FileInfo, err error) error {
			if cf.Chainstore.Path == path {
				return nil // skip the root
			}
			if err = os.RemoveAll(path); err != nil {
				return fmt.Errorf("Failed to remove or clean the directory: %s, because: %s", path, err)
			}
			return nil
		},
	)
	if err != nil {
		return nil, err
	}

	// Build the stores and setup the chain
	memStore := metricsmgr.New("fn.store.mem",
		memstore.New(cf.Chainstore.MemCacheSize*1024*1024),
	)

	diskStore := lrumgr.New(cf.Chainstore.DiskCacheSize*1024*1024,
		metricsmgr.New("fn.store.bolt",
			boltstore.New(cf.Chainstore.Path+"store.db", "imgry"),
		),
	)

	var store chainstore.Store

	if cf.Chainstore.S3AccessKey != "" && cf.Chainstore.S3SecretKey != "" {
		s3Store := metricsmgr.New("fn.store.s3",
			s3store.New(cf.Chainstore.S3Bucket, cf.Chainstore.S3AccessKey, cf.Chainstore.S3SecretKey),
		)

		// store = chainstore.New(memStore, chainstore.Async(diskStore, s3Store))
		store = chainstore.New(memStore, chainstore.Async(nil, s3Store))
	} else {
		store = chainstore.New(memStore, chainstore.Async(nil, diskStore))
	}

	if err := store.Open(); err != nil {
		return nil, err
	}
	return store, nil
}

func TestFileStore(t *testing.T) {
	var store chainstore.Store
	var err error

	Convey("Fsdb Open", t, func() {
		store = filestore.New(chainstore.TempDir(), 0755)
		err = nil
		So(err, ShouldEqual, nil)

		Convey("Put/Get/Del basic data", func() {
			err = store.Put("test.txt", []byte{1, 2, 3, 4})
			So(err, ShouldEqual, nil)

			data, err := store.Get("test.txt")
			So(err, ShouldEqual, nil)
			So(data, ShouldResemble, []byte{1, 2, 3, 4})
		})

		Convey("Auto-creating directories on put", func() {
			err = store.Put("hello/there/everyone.txt", []byte{1, 2, 3, 4})
			So(err, ShouldEqual, nil)
		})

	})
}

// TestMockStoreCancelWithDefaultTimeout tests automatic operation cancellation
// and posterior recover.
func TestMockStoreCancelWithDefaultTimeout(t *testing.T) {
	var store chainstore.Store
	var err error

	ctx := context.Background()

	cfg := &Config{
		Capacity:    100,
		SuccessRate: 1.0,             // always succeeds.
		Delay:       time.Second * 1, // any operation takes 0.5s.
	}

	store = chainstore.New(New(cfg))

	assert := assert.New(t)

	// This is going to fail because the timeout is lower than the delay.
	cfg.Timeout = time.Millisecond * 500

	err = store.Put(ctx, "notnil", []byte("something"))
	assert.Equal(chainstore.ErrTimeout, err)

	// This is going to succeed because the timeout is greater than the delay.
	cfg.Timeout = time.Millisecond * 1500

	err = store.Put(ctx, "notnil", []byte("something"))
	assert.Nil(err)

	// This is going to fail because the timeout is lower than the delay.
	cfg.Timeout = time.Millisecond * 500

	_, err = store.Get(ctx, "notnil")
	assert.Equal(chainstore.ErrTimeout, err)

	// This is going to succeed because the timeout is greater than the delay.
	cfg.Timeout = time.Millisecond * 1500

	_, err = store.Get(ctx, "notnil")
	assert.Nil(err)
}

func TestMemCacheStore(t *testing.T) {
	var store chainstore.Store
	store = memstore.New(10)

	Convey("MemCacheStore", t, func() {
		e := store.Put("hi", []byte{1, 2, 3})
		So(e, ShouldEqual, nil)

		obj, e := store.Get("hi")
		So(e, ShouldEqual, nil)
		So(obj, ShouldResemble, []byte{1, 2, 3})

		e = store.Put("bye", []byte{5, 6, 7, 8, 9, 10, 11, 12})
		So(e, ShouldEqual, nil)

		obj, e = store.Get("hi")
		So(len(obj), ShouldEqual, 0)
		obj, e = store.Get("bye")
		So(len(obj), ShouldEqual, 8)
	})

}

// TestMockStoreCancelWithTimeout creates and test a mockstore with that after
// a while gets cancelled.
func TestMockStoreCancelWithTimeout(t *testing.T) {
	var store chainstore.Store
	var err error

	assert := assert.New(t)

	store = chainstore.New(New(&Config{
		Capacity:    100,
		SuccessRate: 1.0,             // always succeeds.
		Delay:       time.Second * 1, // any operation takes 1s.
	}))

	ctx, _ := context.WithTimeout(context.Background(), time.Millisecond*500)

	// After 0.5s this all is going to fail, because the context timed out.
	err = store.Put(ctx, "notnil", []byte("something"))
	assert.NotNil(err)

	err = store.Put(ctx, "hi", []byte{1, 2, 3})
	assert.NotNil(err)

	_, err = store.Get(ctx, "hi")
	assert.NotNil(err)

	err = store.Put(ctx, "bye", []byte{5, 6, 7, 8, 9, 10, 11, 12})
	assert.NotNil(err)

	err = store.Del(ctx, "hi")
	assert.NotNil(err)

	_, err = store.Get(ctx, "hi")
	assert.NotNil(err)

	_, err = store.Get(ctx, "bye")
	assert.NotNil(err)

	_, err = store.Get(ctx, "notnil")
	assert.NotNil(err)
}

// TestMockStoreFail creates and test a mockstore that always fails.
func TestMockStoreFail(t *testing.T) {
	var store chainstore.Store
	var err error

	ctx := context.Background()

	store = chainstore.New(New(&Config{
		Capacity:    100,
		SuccessRate: 0.0, // always succeeds.
	}))

	assert := assert.New(t)

	err = store.Put(ctx, "notnil", []byte("something"))
	assert.NotNil(err)

	err = store.Put(ctx, "hi", []byte{1, 2, 3})
	assert.NotNil(err)

	_, err = store.Get(ctx, "hi")
	assert.NotNil(err)

	err = store.Put(ctx, "bye", []byte{5, 6, 7, 8, 9, 10, 11, 12})
	assert.NotNil(err)

	err = store.Del(ctx, "hi")
	assert.NotNil(err)

	_, err = store.Get(ctx, "hi")
	assert.NotNil(err)

	_, err = store.Get(ctx, "bye")
	assert.NotNil(err)

	_, err = store.Get(ctx, "notnil")
	assert.NotNil(err)
}

func TestBasicChain(t *testing.T) {
	var ms, fs, chain chainstore.Store
	var err error

	logger := log.New(os.Stdout, "", log.LstdFlags)

	Convey("Basic chain", t, func() {
		storeDir := chainstore.TempDir()
		err = nil

		ms = memstore.New(100)
		fs = filestore.New(storeDir+"/filestore", 0755)

		chain = chainstore.New(
			logmgr.New(logger, ""),
			ms,
			fs,
		)
		err = chain.Open()
		So(err, ShouldEqual, nil)

		Convey("Put/Get/Del", func() {
			v := []byte("value")
			err = chain.Put("k", v)
			So(err, ShouldEqual, nil)

			val, err := chain.Get("k")
			So(err, ShouldEqual, nil)
			So(v, ShouldResemble, v)

			val, err = ms.Get("k")
			So(err, ShouldEqual, nil)
			So(val, ShouldResemble, v)

			val, err = fs.Get("k")
			So(err, ShouldEqual, nil)
			So(val, ShouldResemble, v)

			err = chain.Del("k")
			So(err, ShouldEqual, nil)

			val, err = fs.Get("k")
			So(err, ShouldEqual, nil)
			So(len(val), ShouldEqual, 0)

			val, err = chain.Get("[email protected]#")
			So(err, ShouldNotBeNil)
		})
	})
}

func TestS3Store(t *testing.T) {
	var store chainstore.Store
	var err error

	ctx := context.Background()

	assert := assert.New(t)

	store = chainstore.New(New(bucketID, accessKey, secretKey))
	err = store.Open()
	assert.Nil(err)
	defer store.Close()

	// Put a bunch of objects
	e1 := store.Put(ctx, "hi", []byte{1, 2, 3})
	e2 := store.Put(ctx, "bye", []byte{4, 5, 6})
	assert.Nil(e1)
	assert.Nil(e2)

	// Get those objects
	v1, _ := store.Get(ctx, "hi")
	v2, _ := store.Get(ctx, "bye")
	assert.Equal(v1, []byte{1, 2, 3})
	assert.Equal(v2, []byte{4, 5, 6})

	// Delete those objects
	e1 = store.Del(ctx, "hi")
	e2 = store.Del(ctx, "bye")
	assert.Equal(e1, nil)
	assert.Equal(e2, nil)
}

func TestBasicChain(t *testing.T) {
	var ms, fs, chain chainstore.Store
	var err error

	ctx := context.Background()

	logger := log.New(os.Stdout, "", log.LstdFlags)

	storeDir := tempDir()
	err = nil

	ms = memstore.New(100)
	fs = filestore.New(storeDir+"/filestore", 0755)

	chain = chainstore.New(
		logmgr.New(logger, ""),
		ms,
		fs,
	)

	assert := assert.New(t)

	err = chain.Open()
	assert.Nil(err)

	v := []byte("value")
	err = chain.Put(ctx, "k", v)
	assert.Nil(err)

	val, err := chain.Get(ctx, "k")
	assert.Nil(err)
	assert.Equal(val, v)

	val, err = ms.Get(ctx, "k")
	assert.Nil(err)
	assert.Equal(val, v)

	val, err = fs.Get(ctx, "k")
	assert.Nil(err)
	assert.Equal(val, v)

	err = chain.Del(ctx, "k")
	assert.Nil(err)

	val, err = fs.Get(ctx, "k")
	assert.Nil(err)
	assert.Equal(len(val), 0)

	val, err = chain.Get(ctx, "[email protected]#")
	assert.NotNil(err)
}

func TestMetricsMgrStore(t *testing.T) {
	var store chainstore.Store
	var err error

	ctx := context.Background()

	assert := assert.New(t)

	store = chainstore.New(New("ns"))
	err = store.Open()
	assert.Nil(err)
	defer store.Close()

	// Put a bunch of objects
	e1 := store.Put(ctx, "hi", []byte{1, 2, 3})
	e2 := store.Put(ctx, "bye", []byte{4, 5, 6})
	assert.Nil(e1)
	assert.Nil(e2)

	// Delete those objects
	e1 = store.Del(ctx, "hi")
	e2 = store.Del(ctx, "bye")
	assert.Equal(e1, nil)
	assert.Equal(e2, nil)
}

func TestBoltStore(t *testing.T) {
	var store chainstore.Store
	var err error

	store = boltstore.New(chainstore.TempDir()+"/test.db", "test")
	err = store.Open()
	if err != nil {
		t.Error(err)
	}
	defer store.Close() // does this get called?

	Convey("Boltdb Open", t, func() {

		Convey("Put a bunch of objects", func() {
			e1 := store.Put("hi", []byte{1, 2, 3})
			e2 := store.Put("bye", []byte{4, 5, 6})
			So(e1, ShouldEqual, nil)
			So(e2, ShouldEqual, nil)
		})

		Convey("Get those objects", func() {
			v1, _ := store.Get("hi")
			v2, _ := store.Get("bye")
			So(v1, ShouldResemble, []byte{1, 2, 3})
			So(v2, ShouldResemble, []byte{4, 5, 6})
		})

		Convey("Delete those objects", func() {
			e1 := store.Del("hi")
			e2 := store.Del("bye")
			So(e1, ShouldEqual, nil)
			So(e2, ShouldEqual, nil)

			v, _ := store.Get("hi")
			So(len(v), ShouldEqual, 0)
		})

	})
}

func TestAsyncChain(t *testing.T) {
	var ms, fs, bs, chain chainstore.Store
	var err error

	logger := log.New(os.Stdout, "", log.LstdFlags)

	Convey("Async chain", t, func() {
		storeDir := chainstore.TempDir()
		err = nil

		ms = memstore.New(100)
		fs = filestore.New(storeDir+"/filestore", 0755)
		bs = boltstore.New(storeDir+"/boltstore/bolt.db", "test")

		chain = chainstore.New(
			logmgr.New(logger, ""),
			ms,
			chainstore.Async(
				logmgr.New(logger, "async"),
				metricsmgr.New("chaintest", nil,
					fs,
					lrumgr.New(100, bs),
				),
			),
		)
		err = chain.Open()
		So(err, ShouldEqual, nil)

		Convey("Put/Get/Del", func() {
			v := []byte("value")
			err = chain.Put("k", v)
			So(err, ShouldEqual, nil)

			val, err := chain.Get("k")
			So(err, ShouldEqual, nil)
			So(v, ShouldResemble, v)

			val, err = ms.Get("k")
			So(err, ShouldEqual, nil)
			So(val, ShouldResemble, v)

			time.Sleep(10e6) // wait for async operation..

			val, err = fs.Get("k")
			So(err, ShouldEqual, nil)
			So(val, ShouldResemble, v)

			val, err = bs.Get("k")
			So(err, ShouldEqual, nil)
			So(val, ShouldResemble, v)
		})
	})

}

func TestLevelStore(t *testing.T) {
	var store chainstore.Store
	var err error

	store = levelstore.New(chainstore.TempDir())
	err = store.Open()
	if err != nil {
		t.Error(err)
	}
	defer store.Close()

	Convey("Leveldb Open", t, func() {

		Convey("Put a bunch of objects", func() {
			e1 := store.Put("hi", []byte{1, 2, 3})
			e2 := store.Put("bye", []byte{4, 5, 6})
			So(e1, ShouldEqual, nil)
			So(e2, ShouldEqual, nil)
		})

		Convey("Get those objects", func() {
			v1, _ := store.Get("hi")
			v2, _ := store.Get("bye")
			So(v1, ShouldResemble, []byte{1, 2, 3})
			So(v2, ShouldResemble, []byte{4, 5, 6})
		})

		Convey("Delete those objects", func() {
			e1 := store.Del("hi")
			e2 := store.Del("bye")
			So(e1, ShouldEqual, nil)
			So(e2, ShouldEqual, nil)
		})

	})
}