Golang sorted.NewMemoryKeyValue函數代碼示例

func TestBuffer(t *testing.T) {
	var (
		toBack = []mod{
			{false, "b", "b1"},
			{false, "d", "d1"},
			{false, "f", "f1"},
		}
		toBuf = []mod{
			{false, "a", "a2"},
			{false, "b", "b2"},
			{false, "c", "c2"},
			{false, "e", "e2"},
			{true, "f", ""},
			{false, "g", "g2"},
		}
		backBeforeFlush = []mod{
			{false, "b", "b1"},
			{false, "d", "d1"},
			// f deleted
		}
		want = []mod{
			{false, "a", "a2"},
			{false, "b", "b2"},
			{false, "c", "c2"},
			{false, "d", "d1"},
			{false, "e", "e2"},
			// f deleted
			{false, "g", "g2"},
		}
	)

	// Populate backing storage.
	backing := sorted.NewMemoryKeyValue()
	for _, m := range toBack {
		backing.Set(m.key, m.value)
	}
	// Wrap with buffered storage, populate.
	buf := New(sorted.NewMemoryKeyValue(), backing, 1<<20)
	for _, m := range toBuf {
		if m.isDelete {
			buf.Delete(m.key)
		} else {
			buf.Set(m.key, m.value)
		}
	}

	// Check contents of buffered storage.
	check(t, buf, "buffered", want)
	check(t, backing, "backing before flush", backBeforeFlush)

	// Flush.
	if err := buf.Flush(); err != nil {
		t.Fatal("flush error: ", err)
	}

	// Check contents of backing storage.
	check(t, backing, "backing after flush", want)
}

func BenchmarkCorpusFromStorage(b *testing.B) {
	defer test.TLog(b)()
	buildKvOnce.Do(func() {
		kvForBenchmark = sorted.NewMemoryKeyValue()
		idx := index.New(kvForBenchmark)
		id := indextest.NewIndexDeps(idx)
		id.Fataler = b
		for i := 0; i < 10; i++ {
			fileRef, _ := id.UploadFile("file.txt", fmt.Sprintf("some file %d", i), time.Unix(1382073153, 0))
			pn := id.NewPlannedPermanode(fmt.Sprint(i))
			id.SetAttribute(pn, "camliContent", fileRef.String())
		}
	})
	defer index.SetVerboseCorpusLogging(true)
	index.SetVerboseCorpusLogging(false)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		_, err := index.NewCorpusFromStorage(kvForBenchmark)
		if err != nil {
			b.Fatal(err)
		}
	}
}

// NewMemoryIndex returns an Index backed only by memory, for use in tests.
func NewMemoryIndex() *Index {
	ix, err := New(sorted.NewMemoryKeyValue())
	if err != nil {
		// Nothing to fail in memory, so worth panicing about
		// if we ever see something.
		panic(err)
	}
	return ix
}

// TODO(mpl): move this test into kvtest. But that might require
// kvtest taking a "func () sorted.KeyValue) constructor param,
// so kvtest can create several and close in different ways.
func TestMemoryKV_DoubleClose(t *testing.T) {
	kv := sorted.NewMemoryKeyValue()

	it := kv.Find("", "")
	it.Close()
	it.Close()

	kv.Close()
	kv.Close()
}

func TestStorage(t *testing.T) {
	storagetest.Test(t, func(t *testing.T) (sto blobserver.Storage, cleanup func()) {
		s := &storage{
			small: new(test.Fetcher),
			large: new(test.Fetcher),
			meta:  sorted.NewMemoryKeyValue(),
			log:   test.NewLogger(t, "blobpacked: "),
		}
		s.init()
		return s, func() {}
	})
}

func TestIndexingClaimMissingPubkey(t *testing.T) {
	s := sorted.NewMemoryKeyValue()
	idx, err := index.New(s)
	if err != nil {
		t.Fatal(err)
	}

	id := indextest.NewIndexDeps(idx)
	id.Fataler = t

	goodKeyFetcher := id.Index.KeyFetcher
	emptyFetcher := new(test.Fetcher)

	pn := id.NewPermanode()

	// Prevent the index from being able to find the public key:
	idx.KeyFetcher = emptyFetcher

	// This previous failed to upload, since the signer's public key was
	// unavailable.
	claimRef := id.SetAttribute(pn, "tag", "foo")

	t.Logf(" Claim is %v", claimRef)
	t.Logf("Signer is %v", id.SignerBlobRef)

	// Verify that populateClaim noted the missing public key blob:
	{
		key := fmt.Sprintf("missing|%s|%s", claimRef, id.SignerBlobRef)
		if got, err := s.Get(key); got == "" || err != nil {
			t.Errorf("key %q missing (err: %v); want 1", key, err)
		}
	}

	// Now make it available again:
	idx.KeyFetcher = idx.Exp_BlobSource()

	if err := copyBlob(id.SignerBlobRef, idx.Exp_BlobSource().(*test.Fetcher), goodKeyFetcher); err != nil {
		t.Errorf("Error copying public key to BlobSource: %v", err)
	}
	if err := copyBlob(id.SignerBlobRef, idx, goodKeyFetcher); err != nil {
		t.Errorf("Error uploading public key to indexer: %v", err)
	}

	idx.Exp_AwaitReindexing(t)

	// Verify that populateClaim noted the missing public key blob:
	{
		key := fmt.Sprintf("missing|%s|%s", claimRef, id.SignerBlobRef)
		if got, err := s.Get(key); got != "" || err == nil {
			t.Errorf("row %q still exists", key)
		}
	}
}

func TestOutOfOrderIndexing(t *testing.T) {
	tf := new(test.Fetcher)
	s := sorted.NewMemoryKeyValue()

	ix, err := index.New(s)
	if err != nil {
		t.Fatal(err)
	}
	ix.BlobSource = tf

	t.Logf("file ref = %v", fileBlobRef)
	t.Logf("missing data chunks = %v, %v, %v", chunk1ref, chunk2ref, chunk3ref)

	add := func(b *test.Blob) {
		tf.AddBlob(b)
		if _, err := ix.ReceiveBlob(b.BlobRef(), b.Reader()); err != nil {
			t.Fatalf("ReceiveBlob(%v): %v", b.BlobRef(), err)
		}
	}

	add(fileBlob)

	{
		key := fmt.Sprintf("missing|%s|%s", fileBlobRef, chunk1ref)
		if got, err := s.Get(key); got == "" || err != nil {
			t.Errorf("key %q missing (err: %v); want 1", key, err)
		}
	}

	add(chunk1)
	add(chunk2)

	ix.Exp_AwaitReindexing(t)

	{
		key := fmt.Sprintf("missing|%s|%s", fileBlobRef, chunk3ref)
		if got, err := s.Get(key); got == "" || err != nil {
			t.Errorf("key %q missing (err: %v); want 1", key, err)
		}
	}

	add(chunk3)

	ix.Exp_AwaitReindexing(t)

	foreachSorted(t, s, func(k, v string) {
		if strings.HasPrefix(k, "missing|") {
			t.Errorf("Shouldn't have missing key: %q", k)
		}
	})
}

func TestStorageNoSmallSubfetch(t *testing.T) {
	storagetest.Test(t, func(t *testing.T) (sto blobserver.Storage, cleanup func()) {
		s := &storage{
			// We need to hide SubFetcher, to test *storage's SubFetch, as it delegates
			// to the underlying SubFetcher, if small implements that interface.
			small: hideSubFetcher(new(test.Fetcher)),
			large: new(test.Fetcher),
			meta:  sorted.NewMemoryKeyValue(),
			log:   test.NewLogger(t, "blobpacked: "),
		}
		s.init()
		return s, func() {}
	})
}

func testStreamBlobs(t *testing.T,
	small blobserver.Storage,
	large subFetcherStorage,
	populate func(*testing.T, *storage) []storagetest.StreamerTestOpt) {
	s := &storage{
		small: small,
		large: large,
		meta:  sorted.NewMemoryKeyValue(),
		log:   test.NewLogger(t, "blobpacked: "),
	}
	s.init()
	wants := populate(t, s)
	storagetest.TestStreamer(t, s, wants...)
}

// see if storage proxies through to small for Fetch, Stat, and Enumerate.
func TestSmallFallback(t *testing.T) {
	small := new(test.Fetcher)
	s := &storage{
		small: small,
		large: new(test.Fetcher),
		meta:  sorted.NewMemoryKeyValue(),
		log:   test.NewLogger(t, "blobpacked: "),
	}
	s.init()
	b1 := &test.Blob{"foo"}
	b1.MustUpload(t, small)
	wantSB := b1.SizedRef()

	// Fetch
	rc, _, err := s.Fetch(b1.BlobRef())
	if err != nil {
		t.Errorf("failed to Get blob: %v", err)
	} else {
		rc.Close()
	}

	// Stat.
	sb, err := blobserver.StatBlob(s, b1.BlobRef())
	if err != nil {
		t.Errorf("failed to Stat blob: %v", err)
	} else if sb != wantSB {
		t.Errorf("Stat = %v; want %v", sb, wantSB)
	}

	// Enumerate
	saw := false
	ctx, cancel := context.WithCancel(context.TODO())
	defer cancel()
	if err := blobserver.EnumerateAll(ctx, s, func(sb blob.SizedRef) error {
		if sb != wantSB {
			return fmt.Errorf("saw blob %v; want %v", sb, wantSB)
		}
		saw = true
		return nil
	}); err != nil {
		t.Errorf("EnuerateAll: %v", err)
	}
	if !saw {
		t.Error("didn't see blob in Enumerate")
	}
}

func TestStreamBlobs(t *testing.T) {
	small := new(test.Fetcher)
	s := &storage{
		small: small,
		large: new(test.Fetcher),
		meta:  sorted.NewMemoryKeyValue(),
		log:   test.NewLogger(t, "blobpacked: "),
	}
	s.init()

	all := map[blob.Ref]bool{}
	const nBlobs = 10
	for i := 0; i < nBlobs; i++ {
		b := &test.Blob{strconv.Itoa(i)}
		b.MustUpload(t, small)
		all[b.BlobRef()] = true
	}
	ctx, cancel := context.WithCancel(context.TODO())
	defer cancel()
	token := "" // beginning

	got := map[blob.Ref]bool{}
	dest := make(chan blobserver.BlobAndToken, 16)
	done := make(chan bool)
	go func() {
		defer close(done)
		for bt := range dest {
			got[bt.Blob.Ref()] = true
		}
	}()
	err := s.StreamBlobs(ctx, dest, token)
	if err != nil {
		t.Fatalf("StreamBlobs = %v", err)
	}
	<-done
	if !reflect.DeepEqual(got, all) {
		t.Errorf("Got blobs %v; want %v", got, all)
	}
	storagetest.TestStreamer(t, s, storagetest.WantN(nBlobs))
}

// dataStores returns the blobserver that stores the instances configurations, and the kv
// store for the instances states.
func dataStores() (blobserver.Storage, sorted.KeyValue, error) {
	if DevHandler {
		return &memory.Storage{}, sorted.NewMemoryKeyValue(), nil
	}
	dataDir := os.Getenv("CAMLI_GCE_DATA")
	if dataDir == "" {
		dataDir = "camli-gce-data"
		log.Printf("data dir not provided as env var CAMLI_GCE_DATA, so defaulting to %v", dataDir)
	}
	blobsDir := filepath.Join(dataDir, "instance-conf")
	if err := os.MkdirAll(blobsDir, 0700); err != nil {
		return nil, nil, err
	}
	instConf, err := localdisk.New(blobsDir)
	if err != nil {
		return nil, nil, err
	}
	instState, err := leveldb.NewStorage(filepath.Join(dataDir, "instance-state"))
	if err != nil {
		return nil, nil, err
	}
	return instConf, instState, nil
}

func main() {
	launchConfig.MaybeDeploy()
	addr := flag.String("addr", defaultListenAddr(), "specify address for server to listen on")
	flag.Parse()

	memkv := sorted.NewMemoryKeyValue()
	if err := memkv.Set("6401800c.camlistore.net.", "159.203.246.79"); err != nil {
		panic(err)
	}
	if err := memkv.Set("camlistore.net.", "104.154.231.160"); err != nil {
		panic(err)
	}
	if err := memkv.Set("www.camlistore.net.", "104.154.231.160"); err != nil {
		panic(err)
	}

	ds := NewDNSServer(memkv)

	log.Printf("serving DNS on %s\n", *addr)
	if err := dns.ListenAndServe(*addr, "udp", ds); err != nil {
		log.Fatal(err)
	}
}

func newTestStorage() *testStorage {
	sto := &storage{
		index: sorted.NewMemoryKeyValue(),
	}
	if err := sto.setKey(testKey); err != nil {
		panic(err)
	}
	ts := &testStorage{
		sto:   sto,
		blobs: new(test.Fetcher),
		meta:  new(test.Fetcher),
	}
	sto.blobs = ts.blobs
	sto.meta = ts.meta
	sto.testRandIV = func() []byte {
		ts.mu.Lock()
		defer ts.mu.Unlock()
		var ret [16]byte
		ts.iv++
		binary.BigEndian.PutUint64(ret[8:], ts.iv)
		return ret[:]
	}
	return ts
}

// NewMemoryIndex returns an Index backed only by memory, for use in tests.
func NewMemoryIndex() *Index {
	return New(sorted.NewMemoryKeyValue())
}