Golang blob.ByRef函數代碼示例

func TestIsolation(t *testing.T) {
	ld := test.NewLoader()
	master, _ := ld.GetStorage("/good-storage/")

	ns1 := newNamespace(t, ld)
	ns2 := newNamespace(t, ld)
	stoMap := map[string]blobserver.Storage{
		"ns1":    ns1,
		"ns2":    ns2,
		"master": master,
	}

	want := func(src string, want ...blob.Ref) {
		if _, ok := stoMap[src]; !ok {
			t.Fatalf("undefined storage %q", src)
		}
		sort.Sort(blob.ByRef(want))
		var got []blob.Ref
		if err := blobserver.EnumerateAll(context.TODO(), stoMap[src], func(sb blob.SizedRef) error {
			got = append(got, sb.Ref)
			return nil
		}); err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(got, want) {
			t.Errorf("server %q = %q; want %q", src, got, want)
		}
	}

	b1 := &test.Blob{Contents: "Blob 1"}
	b1r := b1.BlobRef()
	b2 := &test.Blob{Contents: "Blob 2"}
	b2r := b2.BlobRef()
	b3 := &test.Blob{Contents: "Shared Blob"}
	b3r := b3.BlobRef()

	b1.MustUpload(t, ns1)
	want("ns1", b1r)
	want("ns2")
	want("master", b1r)

	b2.MustUpload(t, ns2)
	want("ns1", b1r)
	want("ns2", b2r)
	want("master", b1r, b2r)

	b3.MustUpload(t, ns2)
	want("ns1", b1r)
	want("ns2", b2r, b3r)
	want("master", b1r, b2r, b3r)

	b3.MustUpload(t, ns1)
	want("ns1", b1r, b3r)
	want("ns2", b2r, b3r)
	want("master", b1r, b2r, b3r)

	if _, _, err := ns2.FetchStreaming(b1r); err == nil {
		t.Errorf("b1 shouldn't be accessible via ns2")
	}
}

func (s *Storage) StreamBlobs(ctx *context.Context, dest chan<- blobserver.BlobAndToken, contToken string) error {
	// for this impl, contToken is >= blobref.String()
	defer close(dest)
	s.mu.RLock()
	defer s.mu.RUnlock()

	sorted := make([]blob.Ref, 0, len(s.m))
	for br := range s.m {
		sorted = append(sorted, br)
	}
	sort.Sort(blob.ByRef(sorted))

	for _, br := range sorted {
		if br.String() < contToken {
			continue
		}
		select {
		case <-ctx.Done():
			return context.ErrCanceled
		case dest <- blobserver.BlobAndToken{
			Blob: blob.NewBlob(br, uint32(len(s.m[br])), func() types.ReadSeekCloser {
				return blob.NewLazyReadSeekCloser(s, br)
			}),
			Token: br.String(),
		}:
		}
	}
	return nil
}

func (s *Storage) EnumerateBlobs(ctx *context.Context, dest chan<- blob.SizedRef, after string, limit int) error {
	defer close(dest)
	s.mu.RLock()
	defer s.mu.RUnlock()

	// TODO(bradfitz): care about keeping this sorted like we used
	// to? I think it was more expensive than it was worth before,
	// since maintaining it was more costly than how often it was
	// used. But perhaps it'd make sense to maintain it lazily:
	// construct it on EnumerateBlobs but invalidate it everywhere
	// else.  Probably doesn't matter much.
	sorted := make([]blob.Ref, 0, len(s.m))
	for br := range s.m {
		sorted = append(sorted, br)
	}
	sort.Sort(blob.ByRef(sorted))

	n := 0
	for _, br := range sorted {
		if after != "" && br.String() <= after {
			continue
		}
		select {
		case dest <- blob.SizedRef{br, uint32(len(s.m[br]))}:
		case <-ctx.Done():
			return context.ErrCanceled
		}
		n++
		if limit > 0 && n == limit {
			break
		}
	}
	return nil
}

// Tests the continue token on recent permanodes, notably when the
// page limit truncates in the middle of a bunch of permanodes with the
// same modtime.
func TestQueryRecentPermanodes_Continue(t *testing.T) {
	testQueryTypes(t, memIndexTypes, func(qt *queryTest) {
		id := qt.id

		var blobs []blob.Ref
		for i := 1; i <= 4; i++ {
			pn := id.NewPlannedPermanode(fmt.Sprint(i))
			blobs = append(blobs, pn)
			t.Logf("permanode %d is %v", i, pn)
			id.SetAttribute_NoTimeMove(pn, "foo", "bar")
		}
		sort.Sort(blob.ByRef(blobs))
		for i, br := range blobs {
			t.Logf("Sorted %d = %v", i, br)
		}
		handler := qt.Handler()

		contToken := ""
		tests := [][]blob.Ref{
			[]blob.Ref{blobs[3], blobs[2]},
			[]blob.Ref{blobs[1], blobs[0]},
			[]blob.Ref{},
		}

		for i, wantBlobs := range tests {
			req := &SearchQuery{
				Constraint: &Constraint{
					Permanode: &PermanodeConstraint{},
				},
				Limit:    2,
				Sort:     UnspecifiedSort,
				Continue: contToken,
			}
			res, err := handler.Query(req)
			if err != nil {
				qt.t.Fatalf("Error on query %d: %v", i+1, err)
			}
			t.Logf("Query %d/%d: continue = %q", i+1, len(tests), res.Continue)
			for i, sb := range res.Blobs {
				t.Logf("  res[%d]: %v", i, sb.Blob)
			}

			var want []*SearchResultBlob
			for _, br := range wantBlobs {
				want = append(want, &SearchResultBlob{Blob: br})
			}
			if !reflect.DeepEqual(res.Blobs, want) {
				gotj, wantj := prettyJSON(res.Blobs), prettyJSON(want)
				t.Fatalf("Query %d: Got blobs:\n%s\nWant:\n%s\n", i+1, gotj, wantj)
			}
			contToken = res.Continue
			haveToken := contToken != ""
			wantHaveToken := (i + 1) < len(tests)
			if haveToken != wantHaveToken {
				t.Fatalf("Query %d: token = %q; want token = %v", i+1, contToken, wantHaveToken)
			}
		}
	})
}

func (sh *SyncHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	if req.Method == "POST" {
		if req.FormValue("mode") == "validate" {
			token := req.FormValue("token")
			if xsrftoken.Valid(token, auth.ProcessRandom(), "user", "runFullValidate") {
				sh.startFullValidation()
				http.Redirect(rw, req, "./", http.StatusFound)
			}
		}
		http.Error(rw, "Bad POST request", http.StatusBadRequest)
		return
	}

	// TODO: remove this lock and instead just call currentStatus,
	// and transition to using that here.
	sh.mu.Lock()
	defer sh.mu.Unlock()
	f := func(p string, a ...interface{}) {
		fmt.Fprintf(rw, p, a...)
	}
	now := time.Now()
	f("<h1>Sync Status (for %s to %s)</h1>", sh.fromName, sh.toName)
	f("<p><b>Current status: </b>%s</p>", html.EscapeString(sh.status))
	if sh.idle {
		return
	}

	f("<h2>Stats:</h2><ul>")
	f("<li>Source: %s</li>", html.EscapeString(storageDesc(sh.from)))
	f("<li>Target: %s</li>", html.EscapeString(storageDesc(sh.to)))
	f("<li>Blobs synced: %d</li>", sh.totalCopies)
	f("<li>Bytes synced: %d</li>", sh.totalCopyBytes)
	f("<li>Blobs yet to copy: %d</li>", len(sh.needCopy))
	f("<li>Bytes yet to copy: %d</li>", sh.bytesRemain)
	if !sh.recentCopyTime.IsZero() {
		f("<li>Most recent copy: %s (%v ago)</li>", sh.recentCopyTime.Format(time.RFC3339), now.Sub(sh.recentCopyTime))
	}
	clarification := ""
	if len(sh.needCopy) == 0 && sh.totalErrors > 0 {
		clarification = "(all since resolved)"
	}
	f("<li>Previous copy errors: %d %s</li>", sh.totalErrors, clarification)
	f("</ul>")

	f("<h2>Validation</h2>")
	if len(sh.vshards) == 0 {
		f("Validation disabled")
		token := xsrftoken.Generate(auth.ProcessRandom(), "user", "runFullValidate")
		f("<form method='POST'><input type='hidden' name='mode' value='validate'><input type='hidden' name='token' value='%s'><input type='submit' value='Start validation'></form>", token)
	} else {
		f("<p>Background scan of source and destination to ensure that the destination has everything the source does, or is at least enqueued to sync.</p>")
		f("<ul>")
		f("<li>Shards complete: %d/%d (%.1f%%)</li>",
			sh.vshardDone,
			len(sh.vshards),
			100*float64(sh.vshardDone)/float64(len(sh.vshards)))
		f("<li>Source blobs seen: %d</li>", sh.vsrcCount)
		f("<li>Source bytes seen: %d</li>", sh.vsrcBytes)
		f("<li>Dest blobs seen: %d</li>", sh.vdestCount)
		f("<li>Dest bytes seen: %d</li>", sh.vdestBytes)
		f("<li>Blobs found missing + fixed: %d</li>", sh.vmissing)
		if len(sh.vshardErrs) > 0 {
			f("<li>Validation errors: %s</li>", sh.vshardErrs)
		}
		f("</ul>")
	}

	if len(sh.copying) > 0 {
		f("<h2>Currently Copying</h2><ul>")
		copying := make([]blob.Ref, 0, len(sh.copying))
		for br := range sh.copying {
			copying = append(copying, br)
		}
		sort.Sort(blob.ByRef(copying))
		for _, br := range copying {
			f("<li>%s</li>\n", sh.copying[br])
		}
		f("</ul>")
	}

	recentErrors := make([]blob.Ref, 0, len(sh.recentErrors))
	for _, br := range sh.recentErrors {
		if _, ok := sh.needCopy[br]; ok {
			// Only show it in the web UI if it's still a problem. Blobs that
			// have since succeeded just confused people.
			recentErrors = append(recentErrors, br)
		}
	}
	if len(recentErrors) > 0 {
		f("<h2>Recent Errors</h2><p>Blobs that haven't successfully copied over yet, and their last errors:</p><ul>")
		for _, br := range recentErrors {
			fail := sh.lastFail[br]
			f("<li>%s: %s: %s</li>\n",
				br,
				fail.when.Format(time.RFC3339),
				html.EscapeString(fail.err.Error()))
		}
		f("</ul>")
	}
}