Golang tiling.TileStore类代码示例

func dumpCommits(store tiling.TileStore, n int) {
	tile, err := store.Get(0, -1)
	if err != nil {
		glog.Fatal("Could not read tile: " + err.Error())
	}

	tileLen := tile.LastCommitIndex() + 1
	commits := tile.Commits[:tileLen]

	if n <= 0 {
		n = tileLen
	}
	startIdx := tileLen - n

	// Keep track of empty traces.
	notEmpty := map[string]bool{}

	for i := startIdx; i < tileLen; i++ {
		count := 0
		for traceKey, v := range tile.Traces {
			if !v.IsMissing(i) {
				count++
				notEmpty[traceKey] = true
			}
		}
		commit := commits[i]

		// This works because a hash is always ascii.
		outHash := commit.Hash[:20]
		fmt.Printf("%v: %5d/%5d : %s : %s \n", time.Unix(commit.CommitTime, 0), count, len(tile.Traces), outHash, commit.Author)
	}

	fmt.Printf("Total Commits   : %d\n", tileLen)
	fmt.Printf("Non-empty traces: %d\n", len(notEmpty))
}

func dumpTileToJSON(store tiling.TileStore, nCommits int, nTraces int, fname string) {
	tile, err := store.Get(0, -1)
	if err != nil {
		glog.Fatal("Could not read tile: " + err.Error())
	}

	newTile := tile
	if (nCommits > 0) || (nTraces > 0) {
		lastIdx := tile.LastCommitIndex()
		if nCommits <= 0 {
			nCommits = lastIdx + 1
		}

		if nTraces <= 0 {
			nTraces = len(tile.Traces)
		}

		commitLen := util.MinInt(nCommits, lastIdx+1)
		startCommit := lastIdx + 1 - commitLen
		newTraces := map[string]tiling.Trace{}
		for key, trace := range tile.Traces {
			for i := startCommit; i <= lastIdx; i++ {
				if !trace.IsMissing(i) {
					newTraces[key] = trace
					break
				}
			}
			if len(newTraces) >= nTraces {
				break
			}
		}

		newCommits := tile.Commits[startCommit:]
		newParamSet := map[string][]string{}
		tiling.GetParamSet(newTraces, newParamSet)

		newTile = &tiling.Tile{
			Traces:    newTraces,
			ParamSet:  newParamSet,
			Commits:   newCommits,
			Scale:     tile.Scale,
			TileIndex: tile.TileIndex,
		}
	}

	result, err := json.Marshal(newTile)
	if err != nil {
		glog.Fatalf("Could not marshal to JSON: %s", err)
	}

	err = ioutil.WriteFile(fname, result, 0644)
	if err != nil {
		glog.Fatalf("Could not write output file %s", err)
	}

	fmt.Printf("Commits included: %d\n", len(newTile.Commits))
	fmt.Printf("Traces included:  %d\n", len(newTile.Traces))
}

func md5Commits(store tiling.TileStore, targetHash string, nCommits int) {
	tile, err := store.Get(0, -1)
	if err != nil {
		glog.Fatal("Could not read tile: " + err.Error())
	}

	tileLen := tile.LastCommitIndex() + 1
	commits := tile.Commits[:tileLen]

	// Find the target index.
	endIdx := -1
	for i, v := range commits {
		if strings.HasPrefix(v.Hash, targetHash) {
			endIdx = i
			break
		}
	}
	if endIdx == -1 {
		glog.Fatalf("Unable to find commit %s", targetHash)
	}

	endIdx++
	startIdx := endIdx - nCommits

	traceKeys := make([]string, 0, len(tile.Traces))
	for k, v := range tile.Traces {
		for i := startIdx; i < endIdx; i++ {
			if !v.IsMissing(i) {
				traceKeys = append(traceKeys, k)
				break
			}
		}
	}
	sort.Strings(traceKeys)

	result := make([][]string, len(traceKeys))
	for i, k := range traceKeys {
		switch trace := tile.Traces[k].(type) {
		case *gtypes.GoldenTrace:
			result[i] = trace.Values[startIdx:endIdx]
		case *types.PerfTrace:
			result[i] = asStringSlice(trace.Values[startIdx:endIdx])
		}
	}

	byteStr, err := getBytes(result)
	if err != nil {
		glog.Fatalf("Unable to serialize to bytes: %s", err.Error())
	}

	md5Hash := fmt.Sprintf("%x", md5.Sum(byteStr))

	fmt.Printf("Commit Range    : %s - %s\n", commits[startIdx].Hash, commits[endIdx-1].Hash)
	fmt.Printf("Hash            : %s\n", md5Hash)
	fmt.Printf("Total     traces: %d\n", len(tile.Traces))
	fmt.Printf("Non-empty traces: %d\n", len(traceKeys))
}

// ValidateDataset validates all the tiles stored in a TileStore.
func ValidateDataset(store tiling.TileStore, verbose, echoHashes bool) bool {
	index := -1
	isValid := true
	// If tilebuilding were instantaneous this might cause a false negative, but it's not.
	oldestTS := time.Now().Unix()

	for {
		tile, err := store.Get(0, index)
		if err != nil {
			fmt.Printf("Failed to Get(0, %d): %s\n", index, err)
			isValid = false
			break
		}
		if verbose {
			fmt.Println("TileIndex:", tile.TileIndex)
			fmt.Println("Tile range:", tile.Commits[0].CommitTime, tile.Commits[len(tile.Commits)-1].CommitTime)
			fmt.Println("Tile range:", time.Unix(tile.Commits[0].CommitTime, 0), time.Unix(tile.Commits[len(tile.Commits)-1].CommitTime, 0))
		}
		// Validate the git hashes in the tile.
		err = validateTile(tile, oldestTS, verbose, echoHashes)
		oldestTS = tile.Commits[0].CommitTime
		if err != nil {
			fmt.Printf("Failed to validate tile %d scale 0: %s\n", index, err)
			isValid = false
			break
		}
		if index > 0 && index != tile.TileIndex {
			fmt.Printf("Tile index inconsistent: index %d != tile.TileIndex %d\n", index, tile.TileIndex)
			isValid = false
			break
		}
		if tile.Scale != 0 {
			fmt.Printf("Tile scale isn't 0: tile.Scale %d\n", tile.Scale)
			isValid = false
			break
		}
		if tile.TileIndex > 0 {
			index = tile.TileIndex - 1
		} else {
			break
		}
	}

	return isValid
}

// Start calculating and reporting statistics on the repo and tiles.
//
// We presume the git.Update(true) is called somewhere else, usually this is done
// in the ingester, so the repo is always as good as the ingested tiles.
func Start(tileStore tiling.TileStore, git *gitinfo.GitInfo) {
	coverage := metrics.NewRegisteredGaugeFloat64("stats.tests.bench_runs_per_changelist", metrics.DefaultRegistry)
	skpLatency := metrics.NewRegisteredTimer("stats.skp.update_latency", metrics.DefaultRegistry)
	commits := metrics.NewRegisteredGauge("stats.commits.total", metrics.DefaultRegistry)

	go func() {
		for _ = range time.Tick(2 * time.Minute) {
			tile, err := tileStore.Get(0, -1)
			if err != nil {
				glog.Warning("Failed to get tile: %s", err)
				continue
			}
			numCommits := tile.LastCommitIndex() + 1
			numTraces := len(tile.Traces)
			total := 0
			for _, tr := range tile.Traces {
				for i := 0; i < numCommits; i++ {
					if !tr.IsMissing(i) {
						total += 1
					}
				}
			}
			cov := float64(total) / float64(numCommits*numTraces)
			glog.Info("Coverage: ", cov)
			coverage.Update(cov)

			last, err := git.LastSkpCommit()
			if err != nil {
				glog.Warning("Failed to read last SKP commit: %s", err)
				continue
			}
			skpLatency.Update(time.Since(last))
			commits.Update(int64(git.NumCommits()))
		}
	}()
}

// singleStep does a single round of alerting.
func singleStep(tileStore tiling.TileStore, issueTracker issues.IssueTracker) {
	latencyBegin := time.Now()
	tile, err := tileStore.Get(0, -1)
	if err != nil {
		glog.Errorf("Alerting: Failed to get tile: %s", err)
		return
	}

	tile, err = trimTile(tile)
	if err != nil {
		glog.Errorf("Alerting: Failed to Trim tile down to size: %s", err)
		return
	}

	summary, err := clustering.CalculateClusterSummaries(tile, CLUSTER_SIZE, CLUSTER_STDDEV, skpOnly)
	if err != nil {
		glog.Errorf("Alerting: Failed to calculate clusters: %s", err)
		return
	}
	fresh := []*types.ClusterSummary{}
	for _, c := range summary.Clusters {
		if math.Abs(c.StepFit.Regression) > clustering.INTERESTING_THRESHHOLD {
			fresh = append(fresh, c)
		}
	}
	old, err := ListFrom(tile.Commits[0].CommitTime)
	if err != nil {
		glog.Errorf("Alerting: Failed to get existing clusters: %s", err)
		return
	}
	glog.Infof("Found %d old", len(old))
	glog.Infof("Found %d fresh", len(fresh))
	updated := CombineClusters(fresh, old)
	for _, c := range updated {
		if c.Status == "" {
			c.Status = "New"
		}
		if err := Write(c); err != nil {
			glog.Errorf("Alerting: Failed to write updated cluster: %s", err)
		}
	}

	current, err := ListFrom(tile.Commits[0].CommitTime)
	if err != nil {
		glog.Errorf("Alerting: Failed to get existing clusters: %s", err)
		return
	}
	count := 0
	for _, c := range current {
		if c.Status == "New" {
			count++
		}
		if issueTracker != nil {
			if err := updateBugs(c, issueTracker); err != nil {
				glog.Errorf("Error retrieving bugs: %s", err)
				return
			}
		} else {
			glog.Infof("Skipping ClusterSummary.Bugs update because apiKey is missing.")
			return
		}
	}
	newClustersGauge.Update(int64(count))
	runsCounter.Inc(1)
	alertingLatency.UpdateSince(latencyBegin)
}