Golang glog.Infof函数代码示例

// processRows reads all the rows from the clusters table and constructs a
// slice of ClusterSummary's from them.
func processRows(rows *sql.Rows, err error) ([]*types.ClusterSummary, error) {
	if err != nil {
		return nil, fmt.Errorf("Failed to read from database: %s", err)
	}
	defer util.Close(rows)

	glog.Infof("Found rows %v", rows)

	ret := []*types.ClusterSummary{}

	for rows.Next() {
		var body string
		var id int64
		if err := rows.Scan(&id, &body); err != nil {
			return nil, fmt.Errorf("Failed to read row from database: %s", err)
		}
		c := &types.ClusterSummary{}
		if err := json.Unmarshal([]byte(body), c); err != nil {
			glog.Errorf("Found invalid JSON in clusters table: %d %s", id, err)
			return nil, fmt.Errorf("Failed to read row from database: %s", err)
		}
		c.ID = id
		glog.Infof("ID: %d", id)
		ret = append(ret, c)
	}

	return ret, nil
}

// UploadFile uploads the specified file to the remote dir in Google Storage. It
// also sets the appropriate ACLs on the uploaded file.
func (gs *GsUtil) UploadFile(fileName, localDir, gsDir string) error {
	localFile := filepath.Join(localDir, fileName)
	gsFile := filepath.Join(gsDir, fileName)
	object := &storage.Object{Name: gsFile}
	f, err := os.Open(localFile)
	if err != nil {
		return fmt.Errorf("Error opening %s: %s", localFile, err)
	}
	defer util.Close(f)
	if _, err := gs.service.Objects.Insert(GS_BUCKET_NAME, object).Media(f).Do(); err != nil {
		return fmt.Errorf("Objects.Insert failed: %s", err)
	}
	glog.Infof("Copied %s to %s", localFile, fmt.Sprintf("gs://%s/%s", GS_BUCKET_NAME, gsFile))

	// All objects uploaded to CT's bucket via this util must be readable by
	// the google.com domain. This will be fine tuned later if required.
	objectAcl := &storage.ObjectAccessControl{
		Bucket: GS_BUCKET_NAME, Entity: "domain-google.com", Object: gsFile, Role: "READER",
	}
	if _, err := gs.service.ObjectAccessControls.Insert(GS_BUCKET_NAME, gsFile, objectAcl).Do(); err != nil {
		return fmt.Errorf("Could not update ACL of %s: %s", object.Name, err)
	}
	glog.Infof("Updated ACL of %s", fmt.Sprintf("gs://%s/%s", GS_BUCKET_NAME, gsFile))

	return nil
}

func _main(ts db.DB) {
	week := time.Hour * 24 * 7
	commits, err := ts.List(time.Now().Add(-week), time.Now())
	if err != nil {
		glog.Errorf("Failed to load commits: %s", err)
		return
	}
	if len(commits) > 50 {
		commits = commits[:50]
	}

	begin := time.Now()
	_, _, err = ts.TileFromCommits(commits)
	if err != nil {
		glog.Errorf("Failed to load Tile: %s", err)
		return
	}
	glog.Infof("Time to load tile: %v", time.Now().Sub(begin))
	// Now load a second time.
	begin = time.Now()
	_, _, err = ts.TileFromCommits(commits)
	if err != nil {
		glog.Errorf("Failed to load Tile: %s", err)
		return
	}
	glog.Infof("Time to load tile the second time: %v", time.Now().Sub(begin))
}

func main() {
	defer common.LogPanic()
	common.Init()

	args := flag.Args()
	if len(args) != 2 {
		flag.Usage()
		os.Exit(1)
	}

	bucket, prefix := args[0], args[1]
	v, err := skiaversion.GetVersion()
	if err != nil {
		glog.Fatal(err)
	}
	glog.Infof("Version %s, built at %s", v.Commit, v.Date)

	if *nsqdAddress == "" {
		glog.Fatal("Missing address of nsqd server.")
	}

	globalEventBus, err := geventbus.NewNSQEventBus(*nsqdAddress)
	if err != nil {
		glog.Fatalf("Unable to connect to NSQ server: %s", err)
	}

	eventBus := eventbus.New(globalEventBus)
	eventBus.SubscribeAsync(event.StorageEvent(bucket, prefix), func(evData interface{}) {
		data := evData.(*event.GoogleStorageEventData)
		glog.Infof("Google Storage notification from bucket\n %s:  %s : %s", data.Updated, data.Bucket, data.Name)
	})
	select {}
}

// processTimeRange calls gs.GetLatestGSDirs to get a list of
func (xformer *pdfXformer) processTimeRange(start time.Time, end time.Time) {
	glog.Infof("Processing time range: (%s, %s)", start.Truncate(time.Second), end.Truncate(time.Second))
	for _, dir := range gs.GetLatestGSDirs(start.Unix(), end.Unix(), *storageJsonDirectory) {
		glog.Infof("> Reading gs://%s/%s\n", *storageBucket, dir)
		requestedObjects := xformer.client.storageService.Objects.List(*storageBucket).Prefix(dir).Fields(
			"nextPageToken", "items/updated", "items/md5Hash", "items/mediaLink", "items/name", "items/metadata")
		for requestedObjects != nil {
			responseObjects, err := requestedObjects.Do()
			if err != nil {
				glog.Errorf("request %#v failed: %s", requestedObjects, err)
			} else {
				for _, jsonObject := range responseObjects.Items {
					xformer.counter++
					glog.Infof("> > Processing object:  gs://%s/%s {%d}", *storageBucket, jsonObject.Name, xformer.counter)
					xformer.processJsonFile(jsonObject)
				}
			}
			if len(responseObjects.NextPageToken) > 0 {
				requestedObjects.PageToken(responseObjects.NextPageToken)
			} else {
				requestedObjects = nil
			}
		}
	}
	glog.Infof("finished time range.")
}

// get retrieves the metadata file if it's changed and writes it to the correct location.
func get(client *http.Client, cert *cert) error {
	// We aren't using the metadata package here because we need to set/get etags.
	r, err := http.NewRequest("GET", "http://metadata/computeMetadata/v1/project/attributes/"+cert.metadata, nil)
	if err != nil {
		return fmt.Errorf("Failed to create request for metadata: %s", err)
	}
	r.Header.Set("Metadata-Flavor", "Google")
	if cert.etag != "" {
		r.Header.Set("If-None-Match", cert.etag)
	}
	resp, err := client.Do(r)
	if err != nil {
		return fmt.Errorf("Failed to retrieve metadata for %s: %s", cert.metadata, err)
	}
	if resp.StatusCode != 200 {
		if cert.etag != "" && resp.StatusCode == 304 {
			// The etag is set and matches what we've already seen, so the file is
			// unchanged. Note that this can't happen the first time get() is called
			// for each file as etag won't be set, so we'll fall through to the code
			// below in that case.
			glog.Infof("etag unchanged for %s: %s", cert.file, cert.etag)
			return nil
		} else {
			return fmt.Errorf("Unexpected status response: %d: %s", resp.StatusCode, resp.Status)
		}
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("Failed to read metadata for %s: %s", cert.metadata, err)
	}
	glog.Infof("Read body for %s: len %d", cert.file, len(body))

	// Store the etag to be used for the next time through get().
	cert.etag = resp.Header.Get("ETag")

	newMD5Hash := fmt.Sprintf("%x", md5.Sum(body))
	if cert.etag != "" || newMD5Hash != md5File(cert.file) {
		// Write out the file to a temp file then sudo mv it over to the right location.
		f, err := ioutil.TempFile("", "certpoller")
		if err != nil {
			glog.Errorf("Failed to create tmp cert file for %s: %s", cert.metadata, err)
		}
		n, err := f.Write(body)
		if err != nil || n != len(body) {
			return fmt.Errorf("Failed to write cert len(body)=%d, n=%d: %s", len(body), n, err)
		}
		tmpName := f.Name()
		if err := f.Close(); err != nil {
			return fmt.Errorf("Failed to close temporary file: %v", err)
		}
		cmd := exec.Command("sudo", "mv", tmpName, cert.file)
		err = cmd.Run()
		if err != nil {
			return fmt.Errorf("Failed to mv certfile into place for %s: %s", cert.metadata, err)
		}
	}
	glog.Infof("Successfully wrote %s", cert.file)
	return nil
}

func main() {
	common.Init()

	v, err := skiaversion.GetVersion()
	if err != nil {
		glog.Fatal(err)
	}
	glog.Infof("Version %s, built at %s", v.Commit, v.Date)

	if *nsqdAddress == "" {
		glog.Fatal("Missing address of nsqd server.")
	}

	globalEventBus, err := geventbus.NewNSQEventBus(*nsqdAddress)
	if err != nil {
		glog.Fatalf("Unable to connect to NSQ server: %s", err)
	}

	eventBus := eventbus.New(globalEventBus)

	// Send events every so often.
	for _ = range time.Tick(2 * time.Second) {
		evData := &event.GoogleStorageEventData{
			Bucket:  "test-bucket",
			Name:    "test-name",
			Updated: time.Now().String(),
		}
		eventBus.Publish(event.GLOBAL_GOOGLE_STORAGE, evData)
		glog.Infof("Sent Event: %#v ", evData)
	}
}

func main() {
	defer common.LogPanic()
	common.Init()

	args := flag.Args()
	if len(args) != 3 {
		glog.Errorf("Expected arguments: branch target buildID")
		glog.Errorf("i.e.:  git_master-skia razor-userdebug 1772442")
		os.Exit(1)
	}

	// Set the arguments necessary to lookup the git hash.
	branch := args[0]
	target := args[1]
	buildID := args[2]
	glog.Infof("Branch, target, buildID: %s, %s, %s", branch, target, buildID)

	// Set up the oauth client.
	var client *http.Client
	var err error

	// In this case we don't want a backoff transport since the Apiary backend
	// seems to fail a lot, so we basically want to fall back to polling if a
	// call fails.
	transport := &http.Transport{
		Dial: util.DialTimeout,
	}

	if *local {
		// Use a local client secret file to load data.
		client, err = auth.InstalledAppClient(OAUTH_CACHE_FILEPATH, CLIENT_SECRET_FILEPATH,
			transport,
			androidbuildinternal.AndroidbuildInternalScope,
			storage.CloudPlatformScope)
		if err != nil {
			glog.Fatalf("Unable to create installed app oauth client:%s", err)
		}
	} else {
		// Use compute engine service account.
		client = auth.GCEServiceAccountClient(transport)
	}

	f, err := androidbuild.New("/tmp/android-gold-ingest", client)
	if err != nil {
		glog.Fatalf("Failed to construct client: %s", err)
	}
	for {
		r, err := f.Get(branch, target, buildID)
		if err != nil {
			glog.Errorf("Failed to get requested info: %s", err)
			time.Sleep(1 * time.Minute)
			continue
		}
		if r != nil {
			glog.Infof("Successfully found: %#v", *r)
		}

		time.Sleep(1 * time.Minute)
	}
}

func pollAndExecOnce() {
	pending, err := frontend.GetOldestPendingTaskV2()
	if err != nil {
		glog.Error(err)
		return
	}
	task := asPollerTask(pending)
	if task == nil {
		return
	}
	taskName, id := task.GetTaskName(), task.GetCommonCols().Id
	glog.Infof("Preparing to execute task %s %d", taskName, id)
	if err = updateAndBuild(); err != nil {
		glog.Error(err)
		return
	}
	glog.Infof("Executing task %s %d", taskName, id)
	if err = task.Execute(); err == nil {
		glog.Infof("Completed task %s %d", taskName, id)
	} else {
		glog.Errorf("Task %s %d failed: %v", taskName, id, err)
		if !*dryRun {
			if err := updateWebappTaskSetFailed(task); err != nil {
				glog.Error(err)
			}
		}
	}
}

// LogFileInfo logs the FileInfoSLice in human readable form, namely file name and if it is a directory or not
func (s FileInfoSlice) LogFileInfo() {
	glog.Infof("Slice contains %d file elements", len(s))
	for _, fi := range s {
		glog.Infof("Name %s, Is directory: %t", fi.Name(), fi.IsDir())
	}
	glog.Info("End File Infos")
}

// scheduleBuilds finds builders with no pending builds, pops the
// highest-priority builds for each from the queue, and requests builds using
// buildbucket.
func scheduleBuilds(q *build_queue.BuildQueue, bb *buildbucket.Client) error {
	errMsg := "Failed to schedule builds: %v"

	// Get the list of idle buildslaves, update the BuildQueue.
	var wg sync.WaitGroup
	var free []*buildslave
	var freeErr error
	wg.Add(1)
	go func() {
		defer wg.Done()
		free, freeErr = getFreeBuildslaves()
	}()

	var updateErr error
	wg.Add(1)
	go func() {
		defer wg.Done()
		updateErr = q.Update()
	}()

	wg.Wait()
	if updateErr != nil {
		return fmt.Errorf(errMsg, updateErr)
	}
	if freeErr != nil {
		return fmt.Errorf(errMsg, freeErr)
	}

	// Sort the list of idle buildslaves, for saner log viewing.
	sort.Sort(buildslaveSlice(free))

	// Schedule builds on free buildslaves.
	errs := []error{}
	glog.Infof("Free buildslaves:")
	for _, s := range free {
		glog.Infof("\t%s", s.Name)
		build, err := q.Pop(s.Builders)
		if err == build_queue.ERR_EMPTY_QUEUE {
			continue
		}
		if *local {
			glog.Infof("Would schedule: %s @ %s, score = %0.3f", build.Builder, build.Commit[0:7], build.Score)
		} else {
			scheduled, err := bb.RequestBuild(build.Builder, s.Master, build.Commit, build.Repo, build.Author)
			if err != nil {
				errs = append(errs, err)
			} else {
				glog.Infof("Scheduled: %s @ %s, score = %0.3f, id = %s", build.Builder, build.Commit[0:7], build.Score, scheduled.Id)
			}
		}
	}
	if len(errs) > 0 {
		errString := "Got errors when scheduling builds:"
		for _, err := range errs {
			errString += fmt.Sprintf("\n%v", err)
		}
		return fmt.Errorf(errString)
	}
	return nil
}

// rasterizeOnce applies a single rastetizer to the given pdf file.
// If the rasterizer fails, use the errorImage.  If everything
// succeeds, upload the PNG.
func (xformer *pdfXformer) rasterizeOnce(pdfPath string, rasterizerIndex int) (string, error) {
	rasterizer := xformer.rasterizers[rasterizerIndex]
	tempdir := filepath.Dir(pdfPath)
	pngPath := path.Join(tempdir, fmt.Sprintf("%s.%s", rasterizer.String(), PNG_EXT))
	defer removeIfExists(pngPath)
	glog.Infof("> > > > rasterizing with %s", rasterizer)
	err := rasterizer.Rasterize(pdfPath, pngPath)
	if err != nil {
		glog.Warningf("rasterizing %s with %s failed: %s", filepath.Base(pdfPath), rasterizer.String(), err)
		return xformer.errorImageMd5, nil
	}
	md5, err := md5OfFile(pngPath)
	if err != nil {
		return "", err
	}
	f, err := os.Open(pngPath)
	if err != nil {
		return "", err
	}
	defer util.Close(f)
	pngUploadPath := fmt.Sprintf("%s/%s.%s", *storageImagesDirectory, md5, PNG_EXT)
	didUpload, err := uploadFile(xformer.client, f, *storageBucket, pngUploadPath, *accessControlEntity)
	if err != nil {
		return "", err
	}
	if didUpload {
		glog.Infof("> > > > uploaded %s", pngUploadPath)
	}
	return md5, nil
}

// Fetch retrieves the file contents.
//
// Callers must call Close() on the returned io.ReadCloser.
func (b ResultsFileLocation) Fetch() (io.ReadCloser, error) {
	if strings.HasPrefix(b.URI, "file://") {
		return os.Open(b.URI[6:])
	} else {
		for i := 0; i < MAX_URI_GET_TRIES; i++ {
			glog.Infof("Fetching: %s", b.Name)
			request, err := gs.RequestForStorageURL(b.URI)
			if err != nil {
				glog.Warningf("Unable to create Storage MediaURI request: %s\n", err)
				continue
			}
			resp, err := client.Do(request)
			if err != nil {
				glog.Warningf("Unable to retrieve URI while creating file iterator: %s", err)
				continue
			}
			if resp.StatusCode != 200 {
				glog.Errorf("Failed to retrieve: %d  %s", resp.StatusCode, resp.Status)
			}
			glog.Infof("GS FETCH %s", b.URI)
			return resp.Body, nil
		}
		return nil, fmt.Errorf("Failed fetching JSON after %d attempts", MAX_URI_GET_TRIES)
	}
}

// NewIngestionProcess creates a Process for ingesting data.
func NewIngestionProcess(git *gitinfo.GitInfo, tileDir, datasetName string, ri ingester.ResultIngester, config map[string]string, every time.Duration, nCommits int, minDuration time.Duration, statusDir, metricName string) ProcessStarter {
	return func() {
		i, err := ingester.NewIngester(git, tileDir, datasetName, ri, nCommits, minDuration, config, statusDir, metricName)
		if err != nil {
			glog.Fatalf("Failed to create Ingester: %s", err)
		}

		glog.Infof("Starting %s ingester. Run every %s.", datasetName, every.String())

		// oneStep is a single round of ingestion.
		oneStep := func() {
			glog.Infof("Running ingester: %s", datasetName)
			err := i.Update()
			if err != nil {
				glog.Error(err)
			}
			glog.Infof("Finished running ingester: %s", datasetName)
		}

		// Start the ingester.
		go func() {
			oneStep()
			for _ = range time.Tick(every) {
				oneStep()
			}
		}()
	}
}

// waitForAnalysis waits for files that need to be analyzed (from forAnalysis) and makes a copy of
// them in config.Aggregator.BinaryFuzzPath with their hash as a file name.
// It then analyzes it using the supplied AnalysisPackage and then signals the results should be
// uploaded. If any unrecoverable errors happen, this method terminates.
func (agg *BinaryAggregator) waitForAnalysis(identifier int, analysisPackage AnalysisPackage) {
	defer agg.pipelineWaitGroup.Done()
	defer go_metrics.GetOrRegisterCounter("analysis_process_count", go_metrics.DefaultRegistry).Dec(int64(1))
	glog.Infof("Spawning analyzer %d", identifier)

	// our own unique working folder
	executableDir := filepath.Join(config.Aggregator.ExecutablePath, fmt.Sprintf("analyzer%d", identifier))
	if err := analysisPackage.Setup(executableDir); err != nil {
		glog.Errorf("Analyzer %d terminated due to error: %s", identifier, err)
		return
	}
	for {
		select {
		case badBinaryPath := <-agg.forAnalysis:
			atomic.AddInt64(&agg.analysisCount, int64(1))
			err := agg.analysisHelper(executableDir, badBinaryPath, analysisPackage)
			if err != nil {
				glog.Errorf("Analyzer %d terminated due to error: %s", identifier, err)
				return
			}
		case <-agg.pipelineShutdown:
			glog.Infof("Analyzer %d recieved shutdown signal", identifier)
			return
		}
	}
}