Golang go-datastore.NewKey函数代码示例

func (ks *DSSuite) TestBasic(c *C) {
	var size = 1000

	d, err := lru.NewDatastore(size)
	c.Check(err, Equals, nil)

	for i := 0; i < size; i++ {
		err := d.Put(ds.NewKey(strconv.Itoa(i)), i)
		c.Check(err, Equals, nil)
	}

	for i := 0; i < size; i++ {
		j, err := d.Get(ds.NewKey(strconv.Itoa(i)))
		c.Check(j, Equals, i)
		c.Check(err, Equals, nil)
	}

	for i := 0; i < size; i++ {
		err := d.Put(ds.NewKey(strconv.Itoa(i+size)), i)
		c.Check(err, Equals, nil)
	}

	for i := 0; i < size; i++ {
		j, err := d.Get(ds.NewKey(strconv.Itoa(i)))
		c.Check(j, Equals, nil)
		c.Check(err, Equals, ds.ErrNotFound)
	}

	for i := 0; i < size; i++ {
		j, err := d.Get(ds.NewKey(strconv.Itoa(i + size)))
		c.Check(j, Equals, i)
		c.Check(err, Equals, nil)
	}
}

func TestCoalesceSamePutDiffPut(t *testing.T) {
	m := setup()
	done := make(chan struct{})

	go func() {
		m.ds.Put(ds.NewKey("foo"), "bar")
		done <- struct{}{}
	}()
	go func() {
		m.ds.Put(ds.NewKey("foo"), "bar")
		done <- struct{}{}
	}()
	go func() {
		m.ds.Put(ds.NewKey("foo"), "bar2")
		done <- struct{}{}
	}()
	go func() {
		m.ds.Put(ds.NewKey("foo"), "bar3")
		done <- struct{}{}
	}()

	<-done
	<-done
	<-done
	<-done

	if m.inside != 3 {
		t.Error("incalls should be 3", m.inside)
	}

	if m.outside != 4 {
		t.Error("outcalls should be 4", m.outside)
	}
}

func TestCoalesceHas(t *testing.T) {
	m := setup()
	done := make(chan struct{})
	errs := make(chan error, 30)

	m.ds.Put(ds.NewKey("foo1"), "bar")
	m.ds.Put(ds.NewKey("foo2"), "baz")

	for i := 0; i < 10; i++ {
		go func() {
			v, err := m.ds.Has(ds.NewKey("foo1"))
			if err != nil {
				errs <- err
			}
			if !v {
				errs <- fmt.Errorf("should have foo1")
			}
			done <- struct{}{}
		}()
	}
	for i := 0; i < 10; i++ {
		go func() {
			v, err := m.ds.Has(ds.NewKey("foo2"))
			if err != nil {
				errs <- err
			}
			if !v {
				errs <- fmt.Errorf("should have foo2")
			}
			done <- struct{}{}
		}()
	}
	for i := 0; i < 10; i++ {
		go func() {
			v, err := m.ds.Has(ds.NewKey("foo3"))
			if err != nil {
				errs <- err
			}
			if v {
				errs <- fmt.Errorf("should not have foo3")
			}
			done <- struct{}{}
		}()
	}

	for i := 0; i < 30; i++ {
		<-done
	}

	if m.inside != 5 {
		t.Error("incalls should be 3", m.inside)
	}

	if m.outside != 32 {
		t.Error("outcalls should be 30", m.outside)
	}
}

func TestCoalesceSameGet(t *testing.T) {
	m := setup()
	done := make(chan struct{})
	errs := make(chan error, 30)

	m.ds.Put(ds.NewKey("foo1"), "bar")
	m.ds.Put(ds.NewKey("foo2"), "baz")

	for i := 0; i < 10; i++ {
		go func() {
			v, err := m.ds.Get(ds.NewKey("foo1"))
			if err != nil {
				errs <- err
			}
			if v != "bar" {
				errs <- fmt.Errorf("v is not bar", v)
			}
			done <- struct{}{}
		}()
	}
	for i := 0; i < 10; i++ {
		go func() {
			v, err := m.ds.Get(ds.NewKey("foo2"))
			if err != nil {
				errs <- err
			}
			if v != "baz" {
				errs <- fmt.Errorf("v is not baz", v)
			}
			done <- struct{}{}
		}()
	}
	for i := 0; i < 10; i++ {
		go func() {
			_, err := m.ds.Get(ds.NewKey("foo3"))
			if err == nil {
				errs <- fmt.Errorf("no error")
			}
			done <- struct{}{}
		}()
	}

	for i := 0; i < 30; i++ {
		<-done
	}

	if m.inside != 5 {
		t.Error("incalls should be 3", m.inside)
	}

	if m.outside != 32 {
		t.Error("outcalls should be 30", m.outside)
	}
}

func (d *Datastore) lookup(key ds.Key) (ds.Datastore, ds.Key, ds.Key) {
	d.lk.Lock()
	defer d.lk.Unlock()
	for _, m := range d.mounts {
		if m.Prefix.Equal(key) || m.Prefix.IsAncestorOf(key) {
			s := strings.TrimPrefix(key.String(), m.Prefix.String())
			k := ds.NewKey(s)
			return m.Datastore, m.Prefix, k
		}
	}
	return nil, ds.NewKey("/"), key
}

func (ks *DSSuite) TestBasic(c *C) {

	mpds := ds.NewMapDatastore()
	nsds := ns.Wrap(mpds, ds.NewKey("abc"))

	keys := strsToKeys([]string{
		"foo",
		"foo/bar",
		"foo/bar/baz",
		"foo/barb",
		"foo/bar/bazb",
		"foo/bar/baz/barb",
	})

	for _, k := range keys {
		err := nsds.Put(k, []byte(k.String()))
		c.Check(err, Equals, nil)
	}

	for _, k := range keys {
		v1, err := nsds.Get(k)
		c.Check(err, Equals, nil)
		c.Check(bytes.Equal(v1.([]byte), []byte(k.String())), Equals, true)

		v2, err := mpds.Get(ds.NewKey("abc").Child(k))
		c.Check(err, Equals, nil)
		c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true)
	}

	run := func(d ds.Datastore, q dsq.Query) []ds.Key {
		r, err := d.Query(q)
		c.Check(err, Equals, nil)

		e, err := r.Rest()
		c.Check(err, Equals, nil)

		return ds.EntryKeys(e)
	}

	listA := run(mpds, dsq.Query{})
	listB := run(nsds, dsq.Query{})
	c.Check(len(listA), Equals, len(listB))

	// sort them cause yeah.
	sort.Sort(ds.KeySlice(listA))
	sort.Sort(ds.KeySlice(listB))

	for i, kA := range listA {
		kB := listB[i]
		c.Check(nsds.InvertKey(kA), Equals, kB)
		c.Check(kA, Equals, nsds.ConvertKey(kB))
	}
}

// Query implements Datastore.Query
func (d *Datastore) Query(q query.Query) (query.Results, error) {

	results := make(chan query.Result)

	walkFn := func(path string, info os.FileInfo, err error) error {
		// remove ds path prefix
		if strings.HasPrefix(path, d.path) {
			path = path[len(d.path):]
		}

		if !info.IsDir() {
			if strings.HasSuffix(path, ObjectKeySuffix) {
				path = path[:len(path)-len(ObjectKeySuffix)]
			}
			key := ds.NewKey(path)
			entry := query.Entry{Key: key.String(), Value: query.NotFetched}
			results <- query.Result{Entry: entry}
		}
		return nil
	}

	go func() {
		filepath.Walk(d.path, walkFn)
		close(results)
	}()
	r := query.ResultsWithChan(q, results)
	r = query.NaiveQueryApply(q, r)
	return r, nil
}

// Query implements Query, inverting keys on the way back out.
func (d *datastore) Query(q dsq.Query) (dsq.Results, error) {
	qr, err := d.raw.Query(q)
	if err != nil {
		return nil, err
	}

	ch := make(chan dsq.Result)
	go func() {
		defer close(ch)
		defer qr.Close()

		for r := range qr.Next() {
			if r.Error != nil {
				ch <- r
				continue
			}

			k := ds.NewKey(r.Entry.Key)
			if !d.prefix.IsAncestorOf(k) {
				continue
			}

			r.Entry.Key = d.Datastore.InvertKey(k).String()
			ch <- r
		}
	}()

	return dsq.DerivedResults(qr, ch), nil
}

func strsToKeys(strs []string) []ds.Key {
	keys := make([]ds.Key, len(strs))
	for i, s := range strs {
		keys[i] = ds.NewKey(s)
	}
	return keys
}

// ConvertKey returns a B58 encoded Datastore key
// TODO: this is hacky because it encodes every path component. some
// path components may be proper strings already...
func (b58KeyConverter) ConvertKey(dsk ds.Key) ds.Key {
	k := ds.NewKey("/")
	for _, n := range dsk.Namespaces() {
		k = k.ChildString(b58.Encode([]byte(n)))
	}
	return k
}

func BenchmarkPutMany(b *testing.B) {
	b.StopTimer()
	path, err := ioutil.TempDir("/tmp", "boltdbtest")
	db, err := NewBoltDatastore(path, "test")
	if err != nil {
		b.Fatal(err)
	}

	values := make(map[string][]byte)
	for i := 0; i < b.N; i++ {
		values[fmt.Sprint(i)] = []byte(fmt.Sprintf("value number %d", i))
	}

	b.StartTimer()

	data := make(map[ds.Key]interface{})
	for k, v := range values {
		dsk := ds.NewKey(k)
		data[dsk] = v
	}
	err = db.PutMany(data)
	if err != nil {
		b.Fatal(err)
	}
}

func TestBasicPutGet(t *testing.T) {
	path, err := ioutil.TempDir("/tmp", "boltdbtest")
	db, err := NewBoltDatastore(path, "test")
	if err != nil {
		t.Fatal(err)
	}

	dsk := ds.NewKey("test")
	somedata := []byte("some data in the datastore")

	err = db.Put(dsk, somedata)
	if err != nil {
		t.Fatal(err)
	}

	val, err := db.Get(dsk)
	if err != nil {
		t.Fatal(err)
	}

	b, ok := val.([]byte)
	if !ok {
		t.Fatal("Got back invalid typed data")
	}

	if !bytes.Equal(b, somedata) {
		t.Fatal("wrong data")
	}

	err = db.Close()
	if err != nil {
		t.Fatal(err)
	}
}

// Put adds a record to the record.Store. Multiple records
// may be put at once to the same path.
// In networked Stores, this may be a blocking operation.
// Some Stores may enforce strict consistency, others may be
// eventually consistent, and some might simply be best effort.
func (s *Store) Put(ctx cxt.Context, p store.Path, r record.Record) error {
	k := ds.NewKey(string(p))
	m, err := record.Marshal(r)
	if err != nil {
		return err
	}

	return s.ds.Put(k, m)
}

func (bd *boltDatastore) Query(q query.Query) (query.Results, error) {
	qrb := query.NewResultBuilder(q)
	qrb.Process.Go(func(worker goprocess.Process) {
		bd.db.View(func(tx *bolt.Tx) error {

			buck := tx.Bucket(bd.bucketName)
			c := buck.Cursor()

			var prefix []byte
			if qrb.Query.Prefix != "" {
				prefix = []byte(qrb.Query.Prefix)
			}

			cur := 0
			sent := 0
			for k, v := c.Seek(prefix); k != nil; k, v = c.Next() {
				if cur < qrb.Query.Offset {
					cur++
					continue
				}
				if qrb.Query.Limit > 0 && sent >= qrb.Query.Limit {
					break
				}
				dk := ds.NewKey(string(k)).String()
				e := query.Entry{Key: dk}

				if !qrb.Query.KeysOnly {
					buf := make([]byte, len(v))
					copy(buf, v)
					e.Value = buf
				}

				select {
				case qrb.Output <- query.Result{Entry: e}: // we sent it out
					sent++
				case <-worker.Closing(): // client told us to end early.
					break
				}
				cur++
			}

			return nil
		})
	})

	// go wait on the worker (without signaling close)
	go qrb.Process.CloseAfterChildren()

	qr := qrb.Results()
	for _, f := range q.Filters {
		qr = query.NaiveFilter(qr, f)
	}
	for _, o := range q.Orders {
		qr = query.NaiveOrder(qr, o)
	}
	return qr, nil
}

func (mt *mountBatch) lookupBatch(key ds.Key) (ds.Batch, ds.Key, error) {
	mt.lk.Lock()
	defer mt.lk.Unlock()

	child, loc, rest := mt.d.lookup(key)
	t, ok := mt.mounts[loc.String()]
	if !ok {
		bds, ok := child.(ds.Batching)
		if !ok {
			return nil, ds.NewKey(""), ds.ErrBatchUnsupported
		}
		var err error
		t, err = bds.Batch()
		if err != nil {
			return nil, ds.NewKey(""), err
		}
		mt.mounts[loc.String()] = t
	}
	return t, rest, nil
}