Golang expstorage.Expectations类代码示例

// ClosestDigest returns the closest digest of type 'label' to 'digest', or "" if there aren't any positive digests.
//
// If no digest of type 'label' is found then Closest.Digest is the empty string.
func ClosestDigest(test string, digest string, exp *expstorage.Expectations, tallies tally.Tally, diffStore diff.DiffStore, label types.Label) *Closest {
	ret := newClosest()
	unavailableDigests := diffStore.UnavailableDigests()

	if unavailableDigests[digest] {
		return ret
	}

	selected := []string{}
	for d, _ := range tallies {
		if !unavailableDigests[d] && (exp.Classification(test, d) == label) {
			selected = append(selected, d)
		}
	}

	if len(selected) == 0 {
		return ret
	}

	if diffMetrics, err := diffStore.Get(digest, selected); err != nil {
		glog.Errorf("ClosestDigest: Failed to get diff: %s", err)
		return ret
	} else {
		for digest, diff := range diffMetrics {
			if delta := combinedDiffMetric(diff.PixelDiffPercent, diff.MaxRGBADiffs); delta < ret.Diff {
				ret.Digest = digest
				ret.Diff = delta
				ret.DiffPixels = diff.PixelDiffPercent
				ret.MaxRGBA = diff.MaxRGBADiffs
			}
		}
		return ret
	}
}

func buildDetailsGUI(tile *tiling.Tile, exp *expstorage.Expectations, test string, top string, left string, graphs bool, closest bool, includeIgnores bool) *PolyDetailsGUI {
	ret := &PolyDetailsGUI{
		TopStatus:  exp.Classification(test, top).String(),
		LeftStatus: exp.Classification(test, left).String(),
		Params:     []*PerParamCompare{},
		Traces:     []*Trace{},
		TileSize:   len(tile.Commits),
	}

	topParamSet := paramsetSum.Get(test, top, includeIgnores)
	leftParamSet := paramsetSum.Get(test, left, includeIgnores)

	traceNames := []string{}
	tally := tallies.ByTrace()
	for id, tr := range tile.Traces {
		if tr.Params()[types.PRIMARY_KEY_FIELD] == test {
			traceNames = append(traceNames, id)
		}
	}

	keys := util.UnionStrings(util.KeysOfParamSet(topParamSet), util.KeysOfParamSet(leftParamSet))
	sort.Strings(keys)
	for _, k := range keys {
		ret.Params = append(ret.Params, &PerParamCompare{
			Name: k,
			Top:  safeGet(topParamSet, k),
			Left: safeGet(leftParamSet, k),
		})
	}

	// Now build the trace data.
	if graphs {
		ret.Traces, ret.OtherDigests = buildTraceData(top, traceNames, tile, tally, exp)
		ret.Commits = tile.Commits
		ret.Blame = blamer.GetBlame(test, top, ret.Commits)
	}

	// Now find the closest positive and negative digests.
	t := tallies.ByTest()[test]
	if closest && t != nil {
		ret.PosClosest = digesttools.ClosestDigest(test, top, exp, t, storages.DiffStore, types.POSITIVE)
		ret.NegClosest = digesttools.ClosestDigest(test, top, exp, t, storages.DiffStore, types.NEGATIVE)
	}

	if top == left {
		var err error
		// Search is only done on the digest. Codesite can't seem to extract the
		// name of the test reliably from the URL in comment text, yet can get the
		// digest just fine. This issue should be revisited once we switch to
		// Monorail.
		ret.Issues, err = issueTracker.FromQuery(top)
		if err != nil {
			glog.Errorf("Failed to load issues for [%s, %s]: %s", test, top, err)
		}
	}

	return ret
}

func digestFromIntermediate(test, digest string, inter *intermediate, e *expstorage.Expectations, tile *tiling.Tile, tallies *tally.Tallies, blamer *blame.Blamer, diffStore diff.DiffStore, paramset *paramsets.Summary, includeIgnores bool) *Digest {
	traceTally := tallies.ByTrace()
	ret := &Digest{
		Test:   test,
		Digest: digest,
		Status: e.Classification(test, digest).String(),
		Traces: buildTraces(test, digest, inter, e, tile, traceTally, paramset, includeIgnores),
		Diff:   buildDiff(test, digest, inter, e, tile, tallies.ByTest(), blamer, diffStore, paramset, includeIgnores),
	}
	return ret
}

// searchTile queries across a tile.
func searchTile(q *Query, e *expstorage.Expectations, parsedQuery url.Values, storages *storage.Storage, tile *tiling.Tile, tallies *tally.Tallies, blamer *blame.Blamer, paramset *paramsets.Summary) ([]*Digest, []*tiling.Commit, error) {
	// TODO Use CommitRange to create a trimmed tile.

	traceTally := tallies.ByTrace()
	lastCommitIndex := tile.LastCommitIndex()

	// Loop over the tile and pull out all the digests that match
	// the query, collecting the matching traces as you go. Build
	// up a set of intermediate's that can then be used to calculate
	// Digest's.

	// map [test:digest] *intermediate
	inter := map[string]*intermediate{}
	for id, tr := range tile.Traces {
		if tiling.Matches(tr, parsedQuery) {
			test := tr.Params()[types.PRIMARY_KEY_FIELD]
			// Get all the digests
			digests := digestsFromTrace(id, tr, q.Head, lastCommitIndex, traceTally)
			for _, digest := range digests {
				cl := e.Classification(test, digest)
				if q.excludeClassification(cl) {
					continue
				}

				// Fix blamer to make this easier.
				if q.BlameGroupID != "" {
					if cl == types.UNTRIAGED {
						b := blamer.GetBlame(test, digest, tile.Commits)
						if q.BlameGroupID != blameGroupID(b, tile.Commits) {
							continue
						}
					} else {
						continue
					}
				}
				key := fmt.Sprintf("%s:%s", test, digest)
				if i, ok := inter[key]; !ok {
					inter[key] = newIntermediate(test, digest, id, tr, digests)
				} else {
					i.addTrace(id, tr, digests)
				}
			}
		}
	}
	// Now loop over all the intermediates and build a Digest for each one.
	ret := make([]*Digest, 0, len(inter))
	for key, i := range inter {
		parts := strings.Split(key, ":")
		ret = append(ret, digestFromIntermediate(parts[0], parts[1], i, e, tile, tallies, blamer, storages.DiffStore, paramset, q.IncludeIgnores))
	}
	return ret, tile.Commits, nil
}

// searchByIssue searches across the given issue.
func searchByIssue(issue string, q *Query, exp *expstorage.Expectations, parsedQuery url.Values, storages *storage.Storage, tile *tiling.Tile, tallies *tally.Tallies, tileParamSet *paramsets.Summary) ([]*Digest, error) {
	trybotResults, err := storages.TrybotResults.Get(issue)
	if err != nil {
		return nil, err
	}

	if !q.IncludeIgnores {
		matcher, err := storages.IgnoreStore.BuildRuleMatcher()
		if err != nil {
			return nil, fmt.Errorf("Unable to build rules matcher: %s", err)
		}
		for k, v := range trybotResults {
			if _, ok := matcher(v.Params); ok {
				delete(trybotResults, k)
			}
		}
	}

	rule := ignore.NewQueryRule(parsedQuery)

	// Aggregate the results into an intermediate representation to avoid
	// passing over the dataset twice.
	inter := map[string]*issueIntermediate{}
	for _, tbr := range trybotResults {
		if rule.IsMatch(tbr.Params) {
			key := tbr.Test + ":" + tbr.Digest
			if found, ok := inter[key]; ok {
				found.add(tbr)
			} else if cl := exp.Classification(tbr.Test, tbr.Digest); !q.excludeClassification(cl) {
				inter[key] = newIssueIntermediate(tbr, cl)
			}
		}
	}

	// Build the output.
	talliesByTest := tallies.ByTest()
	ret := make([]*Digest, 0, len(inter))
	for _, i := range inter {
		ret = append(ret, &Digest{
			Test:     i.test,
			Digest:   i.digest,
			Status:   i.status.String(),
			ParamSet: i.paramSet,
			Diff:     buildDiff(i.test, i.digest, exp, tile, talliesByTest, nil, storages.DiffStore, tileParamSet, q.IncludeIgnores),
		})
	}

	return ret, nil
}

// buildTraces returns a Trace for the given intermediate.
func buildTraces(test, digest string, inter *intermediate, e *expstorage.Expectations, tile *tiling.Tile, traceTally map[string]tally.Tally) *Traces {
	traceNames := make([]string, 0, len(inter.Traces))
	for id, _ := range inter.Traces {
		traceNames = append(traceNames, id)
	}

	ret := &Traces{
		TileSize: len(tile.Commits),
		Traces:   []Trace{},
		Digests:  []DigestStatus{},
	}

	sort.Strings(traceNames)

	last := tile.LastCommitIndex()
	y := 0
	if len(traceNames) > 0 {
		ret.Digests = append(ret.Digests, DigestStatus{
			Digest: digest,
			Status: e.Classification(test, digest).String(),
		})
	}
	for _, id := range traceNames {
		t, ok := traceTally[id]
		if !ok {
			continue
		}
		if count, ok := t[digest]; !ok || count == 0 {
			continue
		}
		trace := inter.Traces[id].(*types.GoldenTrace)
		p := Trace{
			Data:   []Point{},
			ID:     id,
			Params: trace.Params(),
		}
		for i := last; i >= 0; i-- {
			if trace.IsMissing(i) {
				continue
			}
			// s is the status of the digest, it is either 0 for a match, or [1-8] if not.
			s := 0
			if trace.Values[i] != digest {
				if index := digestIndex(trace.Values[i], ret.Digests); index != -1 {
					s = index
				} else {
					if len(ret.Digests) < 9 {
						d := trace.Values[i]
						ret.Digests = append(ret.Digests, DigestStatus{
							Digest: d,
							Status: e.Classification(test, d).String(),
						})
						s = len(ret.Digests) - 1
					} else {
						s = 8
					}
				}
			}
			p.Data = append(p.Data, Point{
				X: i,
				Y: y,
				S: s,
			})
		}
		sort.Sort(PointSlice(p.Data))
		ret.Traces = append(ret.Traces, p)
		y += 1
	}

	return ret
}

// buildTraceData returns a populated []*Trace for all the traces that contain 'digest'.
func buildTraceData(digest string, traceNames []string, tile *tiling.Tile, traceTally map[string]tally.Tally, exp *expstorage.Expectations) ([]*Trace, []*DigestStatus) {
	sort.Strings(traceNames)
	ret := []*Trace{}
	last := tile.LastCommitIndex()
	y := 0

	// Keep track of the first 7 non-matching digests we encounter so we can color them differently.
	otherDigests := []*DigestStatus{}
	// Populate otherDigests with all the digests, including the one we are comparing against.
	if len(traceNames) > 0 {
		// Find the test name so we can look up the triage status.
		trace := tile.Traces[traceNames[0]].(*types.GoldenTrace)
		test := trace.Params()[types.PRIMARY_KEY_FIELD]
		otherDigests = append(otherDigests, &DigestStatus{
			Digest: digest,
			Status: exp.Classification(test, digest).String(),
		})
	}
	for _, id := range traceNames {
		t, ok := traceTally[id]
		if !ok {
			continue
		}
		if count, ok := t[digest]; !ok || count == 0 {
			continue
		}
		trace := tile.Traces[id].(*types.GoldenTrace)
		p := &Trace{
			Data:   []Point{},
			Label:  id,
			Params: trace.Params(),
		}
		for i := last; i >= 0; i-- {
			if trace.IsMissing(i) {
				continue
			}
			// s is the status of the digest, it is either 0 for a match, or [1-8] if not.
			s := 0
			if trace.Values[i] != digest {
				if index := digestIndex(trace.Values[i], otherDigests); index != -1 {
					s = index
				} else {
					if len(otherDigests) < 9 {
						d := trace.Values[i]
						test := trace.Params()[types.PRIMARY_KEY_FIELD]
						otherDigests = append(otherDigests, &DigestStatus{
							Digest: d,
							Status: exp.Classification(test, d).String(),
						})
						s = len(otherDigests) - 1
					} else {
						s = 8
					}
				}
			}
			p.Data = append(p.Data, Point{
				X: i,
				Y: y,
				S: s,
			})
		}
		sort.Sort(PointSlice(p.Data))
		ret = append(ret, p)
		y += 1
	}

	return ret, otherDigests
}

// searchByIssue searches across the given issue.
func searchByIssue(issue string, q *Query, exp *expstorage.Expectations, parsedQuery url.Values, storages *storage.Storage, tile *tiling.Tile, tallies *tally.Tallies, tileParamSet *paramsets.Summary) ([]*Digest, error) {
	trybotResults, err := storages.TrybotResults.Get(issue)
	if err != nil {
		return nil, err
	}

	// Get a matcher for the ignore rules if we filter ignores.
	var ignoreMatcher ignore.RuleMatcher = nil
	if !q.IncludeIgnores {
		ignoreMatcher, err = storages.IgnoreStore.BuildRuleMatcher()
		if err != nil {
			return nil, fmt.Errorf("Unable to build rules matcher: %s", err)
		}
	}

	// Set up a rule to match the query.
	var queryRule ignore.QueryRule = nil
	if len(parsedQuery) > 0 {
		queryRule = ignore.NewQueryRule(parsedQuery)
	}

	// Aggregate the results into an intermediate representation to avoid
	// passing over the dataset twice.
	inter := map[string]*issueIntermediate{}
	talliesByTest := tallies.ByTest()

	for _, bot := range trybotResults.Bots {
		for _, result := range bot.TestResults {
			expandedParams := util.CopyStringMap(bot.BotParams)
			util.AddParams(expandedParams, result.Params)

			if ignoreMatcher != nil {
				if _, ok := ignoreMatcher(expandedParams); ok {
					continue
				}
			}

			if (queryRule == nil) || queryRule.IsMatch(expandedParams) {
				testName := expandedParams[types.PRIMARY_KEY_FIELD]
				digest := trybotResults.Digests[result.DigestIdx]
				key := testName + ":" + digest
				if !q.IncludeMaster {
					if _, ok := talliesByTest[testName][digest]; ok {
						continue
					}
				}
				if found, ok := inter[key]; ok {
					found.add(expandedParams)
				} else if cl := exp.Classification(testName, digest); !q.excludeClassification(cl) {
					inter[key] = newIssueIntermediate(expandedParams, digest, cl)
				}
			}
		}
	}

	// Build the output and make sure the digest are cached on disk.
	digests := make(map[string]bool, len(inter))
	ret := make([]*Digest, 0, len(inter))
	emptyTraces := &Traces{}
	for _, i := range inter {
		ret = append(ret, &Digest{
			Test:     i.test,
			Digest:   i.digest,
			Status:   i.status.String(),
			ParamSet: i.paramSet,
			Diff:     buildDiff(i.test, i.digest, exp, tile, talliesByTest, nil, storages.DiffStore, tileParamSet, q.IncludeIgnores),
			Traces:   emptyTraces,
		})
		digests[i.digest] = true
	}

	// This ensures that all digests are cached on disk.
	storages.DiffStore.AbsPath(util.KeysOfStringSet(digests))
	return ret, nil
}