Golang context.New函数代码示例

func TestEnumerate(t *testing.T) {
	ds := NewStorage(t)
	defer cleanUp(ds)

	// For test simplicity foo, bar, and baz all have ascending
	// sha1s and lengths.
	foo := &test.Blob{"foo"}   // 0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33
	bar := &test.Blob{"baar"}  // b23361951dde70cb3eca44c0c674181673a129dc
	baz := &test.Blob{"bazzz"} // e0eb17003ce1c2812ca8f19089fff44ca32b3710
	foo.MustUpload(t, ds)
	bar.MustUpload(t, ds)
	baz.MustUpload(t, ds)

	limit := 5000
	ch := make(chan blob.SizedRef)
	errCh := make(chan error)
	go func() {
		errCh <- ds.EnumerateBlobs(context.New(), ch, "", limit)
	}()

	var (
		sb blob.SizedRef
		ok bool
	)
	sb, ok = <-ch
	Assert(t, ok, "got 1st blob")
	ExpectInt(t, 3, int(sb.Size), "1st blob size")
	sb, ok = <-ch
	Assert(t, ok, "got 2nd blob")
	ExpectInt(t, 4, int(sb.Size), "2nd blob size")
	sb, ok = <-ch
	Assert(t, ok, "got 3rd blob")
	ExpectInt(t, 5, int(sb.Size), "3rd blob size")
	sb, ok = <-ch
	Assert(t, !ok, "got channel close")
	ExpectNil(t, <-errCh, "EnumerateBlobs return value")

	// Now again, but skipping foo's blob
	ch = make(chan blob.SizedRef)
	go func() {
		errCh <- ds.EnumerateBlobs(context.New(),
			ch,
			foo.BlobRef().String(),
			limit)
	}()
	sb, ok = <-ch
	Assert(t, ok, "got 1st blob, skipping foo")
	ExpectInt(t, 4, int(sb.Size), "blob size")
	sb, ok = <-ch
	Assert(t, ok, "got 2nd blob, skipping foo")
	ExpectInt(t, 5, int(sb.Size), "blob size")
	sb, ok = <-ch
	Assert(t, !ok, "got final nil")
	ExpectNil(t, <-errCh, "EnumerateBlobs return value")
}

func (ia *importerAcct) start() {
	ia.mu.Lock()
	defer ia.mu.Unlock()
	if ia.current != nil {
		return
	}
	ctx := context.New()
	rc := &RunContext{
		Context: ctx,
		Host:    ia.im.host,
		ia:      ia,
	}
	ia.current = rc
	ia.stopped = false
	ia.lastRunStart = time.Now()
	go func() {
		log.Printf("Starting importer %s: %s", ia.im.name, ia.AccountLinkSummary())
		err := ia.im.impl.Run(rc)
		if err != nil {
			log.Printf("Importer %s error: %v", ia.im.name, err)
		} else {
			log.Printf("Importer %s finished.", ia.im.name)
		}
		ia.mu.Lock()
		defer ia.mu.Unlock()
		ia.current = nil
		ia.stopped = false
		ia.lastRunDone = time.Now()
		ia.lastRunErr = err
	}()
}

func TestGetUserID(t *testing.T) {
	im := &imp{
		credsVal: &oauth.Credentials{
			Token:  "foo",
			Secret: "bar",
		},
	}
	ctx := context.New()
	ctx.SetHTTPClient(&http.Client{
		Transport: newFakeTransport(map[string]func() *http.Response{
			apiURL + userInfoAPIPath: fileResponder(filepath.FromSlash("testdata/verify_credentials-res.json")),
		}),
	})
	inf, err := im.getUserInfo(ctx)
	if err != nil {
		t.Fatal(err)
	}
	want := userInfo{
		ID:         "2325935334",
		ScreenName: "lejatorn",
		Name:       "Mathieu Lonjaret",
	}
	if inf != want {
		t.Errorf("user info = %+v; want %+v", inf, want)
	}
}

func (ia *importerAcct) start() {
	ia.mu.Lock()
	defer ia.mu.Unlock()
	if ia.current != nil {
		return
	}
	rc := &RunContext{
		// TODO: context plumbing
		Context: context.New(context.WithHTTPClient(ia.im.host.HTTPClient())),
		Host:    ia.im.host,
		ia:      ia,
	}
	ia.current = rc
	ia.stopped = false
	ia.lastRunStart = time.Now()
	go func() {
		log.Printf("Starting %v: %s", ia, ia.AccountLinkSummary())
		err := ia.im.impl.Run(rc)
		if err != nil {
			log.Printf("%v error: %v", ia, err)
		} else {
			log.Printf("%v finished.", ia)
		}
		ia.mu.Lock()
		defer ia.mu.Unlock()
		ia.current = nil
		ia.stopped = false
		ia.lastRunDone = time.Now()
		ia.lastRunErr = err
		go ia.maybeStart()
	}()
}

func checkEnumerate(idx *index.Index, want []blob.SizedRef, args *enumArgs) error {
	if args == nil {
		args = &enumArgs{}
	}
	if args.ctx == nil {
		args.ctx = context.New()
	}
	if args.dest == nil {
		args.dest = make(chan blob.SizedRef)
	}
	if args.limit == 0 {
		args.limit = 5000
	}
	errCh := make(chan error)
	go func() {
		errCh <- idx.EnumerateBlobs(args.ctx, args.dest, args.after, args.limit)
	}()
	for k, sbr := range want {
		got, ok := <-args.dest
		if !ok {
			return fmt.Errorf("could not enumerate blob %d", k)
		}
		if got != sbr {
			return fmt.Errorf("enumeration %d: got %v, wanted %v", k, got, sbr)
		}
	}
	_, ok := <-args.dest
	if ok {
		return errors.New("chan was not closed after enumeration")
	}
	return <-errCh
}

func (sh *SyncHandler) validateShardPrefix(pfx string) (err error) {
	defer func() {
		sh.mu.Lock()
		if err != nil {
			errs := fmt.Sprintf("Failed to validate prefix %s: %v", pfx, err)
			sh.logf("%s", errs)
			sh.vshardErrs = append(sh.vshardErrs, errs)
		} else {
			sh.vshardDone++
		}
		sh.mu.Unlock()
	}()
	ctx := context.New()
	defer ctx.Cancel()
	src, serrc := sh.startValidatePrefix(ctx, pfx, false)
	dst, derrc := sh.startValidatePrefix(ctx, pfx, true)

	missing := make(chan blob.SizedRef, 8)
	go blobserver.ListMissingDestinationBlobs(missing, func(blob.Ref) {}, src, dst)
	for sb := range missing {
		sh.mu.Lock()
		sh.vmissing++
		sh.mu.Unlock()
		// TODO: stats for missing blobs found.
		sh.enqueue(sb)
	}

	if err := <-serrc; err != nil {
		return fmt.Errorf("Error enumerating source %s for validating shard %s: %v", sh.fromName, pfx, err)
	}
	if err := <-derrc; err != nil {
		return fmt.Errorf("Error enumerating target %s for validating shard %s: %v", sh.toName, pfx, err)
	}
	return nil
}

func TestS3(t *testing.T) {
	if *bucket == "" || *key == "" || *secret == "" {
		t.Skip("Skipping test because at least one of -s3_key, -s3_secret, or -s3_bucket flags has not been provided.")
	}
	if !strings.HasPrefix(*bucket, "camlistore-") || !strings.HasSuffix(*bucket, "-test") {
		t.Fatalf("bogus bucket name %q; must begin with 'camlistore-' and end in '-test'", *bucket)
	}
	storagetest.Test(t, func(t *testing.T) (sto blobserver.Storage, cleanup func()) {
		sto, err := newFromConfig(nil, jsonconfig.Obj{
			"aws_access_key":        *key,
			"aws_secret_access_key": *secret,
			"bucket":                *bucket,
		})
		if err != nil {
			t.Fatalf("newFromConfig error: %v", err)
		}
		if !testing.Short() {
			log.Printf("Warning: this test does many serial operations. Without the go test -short flag, this test will be very slow.")
		}
		clearBucket := func() {
			var all []blob.Ref
			blobserver.EnumerateAll(context.New(), sto, func(sb blob.SizedRef) error {
				t.Logf("Deleting: %v", sb.Ref)
				all = append(all, sb.Ref)
				return nil
			})
			if err := sto.RemoveBlobs(all); err != nil {
				t.Fatalf("Error removing blobs during cleanup: %v", err)
			}
		}
		clearBucket()
		return sto, clearBucket
	})
}

func testEnumerate(t *testing.T, sto blobserver.Storage, wantUnsorted []blob.SizedRef, opts ...interface{}) {
	var after string
	var n = 1000
	for _, opt := range opts {
		switch v := opt.(type) {
		case string:
			after = v
		case int:
			n = v
		default:
			panic("bad option of type " + fmt.Sprint("%T", v))
		}
	}

	want := append([]blob.SizedRef(nil), wantUnsorted...)
	sort.Sort(blob.SizedByRef(want))

	sbc := make(chan blob.SizedRef, 10)

	var got []blob.SizedRef
	var grp syncutil.Group
	sawEnd := make(chan bool, 1)
	grp.Go(func() error {
		if err := sto.EnumerateBlobs(context.New(), sbc, after, n); err != nil {
			return fmt.Errorf("EnumerateBlobs(%q, %d): %v", after, n)
		}
		return nil
	})
	grp.Go(func() error {
		for sb := range sbc {
			if !sb.Valid() {
				return fmt.Errorf("invalid blobref %#v received in enumerate", sb)
			}
			got = append(got, sb)
		}
		sawEnd <- true
		return nil

	})
	grp.Go(func() error {
		select {
		case <-sawEnd:
			return nil
		case <-time.After(10 * time.Second):
			return errors.New("timeout waiting for EnumerateBlobs to close its channel")
		}

	})
	if err := grp.Err(); err != nil {
		t.Fatalf("Enumerate error: %v", err)
		return
	}
	if len(got) == 0 && len(want) == 0 {
		return
	}
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("Enumerate mismatch. Got %d; want %d.\n Got: %v\nWant: %v\n",
			len(got), len(want), got, want)
	}
}

func TestEnumerateIsSorted(t *testing.T) {
	ds := NewStorage(t)
	defer cleanUp(ds)

	const blobsToMake = 250
	t.Logf("Uploading test blobs...")
	for i := 0; i < blobsToMake; i++ {
		blob := &test.Blob{fmt.Sprintf("blob-%d", i)}
		blob.MustUpload(t, ds)
	}

	// Make some fake blobs in other partitions to confuse the
	// enumerate code.
	// TODO(bradfitz): remove this eventually.
	fakeDir := ds.root + "/partition/queue-indexer/sha1/1f0/710"
	ExpectNil(t, os.MkdirAll(fakeDir, 0755), "creating fakeDir")
	ExpectNil(t, ioutil.WriteFile(fakeDir+"/sha1-1f07105465650aa243cfc1b1bbb1c68ea95c6812.dat",
		[]byte("fake file"), 0644), "writing fake blob")

	// And the same for a "cache" directory, used by the default configuration.
	fakeDir = ds.root + "/cache/sha1/1f0/710"
	ExpectNil(t, os.MkdirAll(fakeDir, 0755), "creating cache fakeDir")
	ExpectNil(t, ioutil.WriteFile(fakeDir+"/sha1-1f07105465650aa243cfc1b1bbb1c68ea95c6812.dat",
		[]byte("fake file"), 0644), "writing fake blob")

	var tests = []struct {
		limit int
		after string
	}{
		{200, ""},
		{blobsToMake, ""},
		{200, "sha1-2"},
		{200, "sha1-3"},
		{200, "sha1-4"},
		{200, "sha1-5"},
		{200, "sha1-e"},
		{200, "sha1-f"},
		{200, "sha1-ff"},
	}
	for _, test := range tests {
		limit := test.limit
		ch := make(chan blob.SizedRef)
		errCh := make(chan error)
		go func() {
			errCh <- ds.EnumerateBlobs(context.New(), ch, test.after, limit)
		}()
		got := make([]blob.SizedRef, 0, blobsToMake)
		for sb := range ch {
			got = append(got, sb)
		}
		if err := <-errCh; err != nil {
			t.Errorf("case %+v; enumerate error: %v", test, err)
			continue
		}
		if !sort.IsSorted(SortedSizedBlobs(got)) {
			t.Errorf("case %+v: expected sorted; got: %q", test, got)
		}
	}
}

func (sh *SyncHandler) validateShardPrefix(pfx string) (err error) {
	defer func() {
		sh.mu.Lock()
		if err != nil {
			errs := fmt.Sprintf("Failed to validate prefix %s: %v", pfx, err)
			sh.logf("%s", errs)
			sh.vshardErrs = append(sh.vshardErrs, errs)
		} else {
			sh.vshardDone++
		}
		sh.mu.Unlock()
	}()
	ctx := context.New()
	defer ctx.Cancel()
	src, serrc := sh.startValidatePrefix(ctx, pfx, false)
	dst, derrc := sh.startValidatePrefix(ctx, pfx, true)
	srcErr := &chanError{
		C: serrc,
		Wrap: func(err error) error {
			return fmt.Errorf("Error enumerating source %s for validating shard %s: %v", sh.fromName, pfx, err)
		},
	}
	dstErr := &chanError{
		C: derrc,
		Wrap: func(err error) error {
			return fmt.Errorf("Error enumerating target %s for validating shard %s: %v", sh.toName, pfx, err)
		},
	}

	missingc := make(chan blob.SizedRef, 8)
	go blobserver.ListMissingDestinationBlobs(missingc, func(blob.Ref) {}, src, dst)

	var missing []blob.SizedRef
	for sb := range missingc {
		missing = append(missing, sb)
	}

	if err := srcErr.Get(); err != nil {
		return err
	}
	if err := dstErr.Get(); err != nil {
		return err
	}

	for _, sb := range missing {
		if enqErr := sh.enqueue(sb); enqErr != nil {
			if err == nil {
				err = enqErr
			}
		} else {
			sh.mu.Lock()
			sh.vmissing += 1
			sh.mu.Unlock()
		}
	}
	return err
}

func (s *storage) anyZipPacks() (v bool) {
	ctx := context.New()
	defer ctx.Cancel()
	dest := make(chan blob.SizedRef, 1)
	if err := s.large.EnumerateBlobs(ctx, dest, "", 1); err != nil {
		// Not a great interface in general, but only needed
		// by the start-up check for now, where it doesn't
		// really matter.
		return false
	}
	_, ok := <-dest
	return ok
}

func TestEnumerateEmpty(t *testing.T) {
	ds := NewStorage(t)
	defer cleanUp(ds)

	limit := 5000
	ch := make(chan blob.SizedRef)
	errCh := make(chan error)
	go func() {
		errCh <- ds.EnumerateBlobs(context.New(), ch, "", limit)
	}()

	_, ok := <-ch
	Expect(t, !ok, "no first blob")
	ExpectNil(t, <-errCh, "EnumerateBlobs return value")
}

// CreateAccount creates a new importer account for the Host h, and the importer
// implementation named impl. It returns a RunContext setup with that account.
func CreateAccount(h *Host, impl string) (*RunContext, error) {
	imp, ok := h.imp[impl]
	if !ok {
		return nil, fmt.Errorf("host does not have a %v importer", impl)
	}
	ia, err := imp.newAccount()
	if err != nil {
		return nil, fmt.Errorf("could not create new account for importer %v: %v", impl, err)
	}
	return &RunContext{
		// TODO: context plumbing
		Context: context.New(context.WithHTTPClient(ia.im.host.HTTPClient())),
		Host:    ia.im.host,
		ia:      ia,
	}, nil
}

// singleBlob assumes that sto contains a single blob and returns it.
// If there are more or fewer than one blob, it's an error.
func singleBlob(sto blobserver.BlobEnumerator) (ret blob.SizedRef, err error) {
	ctx := context.New()
	defer ctx.Cancel()

	n := 0
	if err = blobserver.EnumerateAll(ctx, sto, func(sb blob.SizedRef) error {
		ret = sb
		n++
		return nil
	}); err != nil {
		return blob.SizedRef{}, err
	}
	if n != 1 {
		return blob.SizedRef{}, fmt.Errorf("saw %d blobs; want 1", n)
	}
	return
}

// It verifies the size and hash of each
// before returning and fails the test if any of the checks fail. It
// also fails the test if StreamBlobs returns a non-nil error.
func streamAll(t *testing.T, s *storage) []*blob.Blob {
	var blobs []*blob.Blob
	ctx := context.New()
	ch := make(chan blobserver.BlobAndToken)
	errCh := make(chan error, 1)

	go func() { errCh <- s.StreamBlobs(ctx, ch, "") }()

	for bt := range ch {
		verifySizeAndHash(t, bt.Blob)
		blobs = append(blobs, bt.Blob)
	}
	if err := <-errCh; err != nil {
		t.Fatalf("StreamBlobs error = %v", err)
	}
	return blobs
}