Golang appengine.Timeout函数代码示例

func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now())
	q = q.Limit(10 * 60 * 2) // 10/s queue, 2 min cron
	it := q.Run(appengine.Timeout(c, time.Minute))
	tc := make(chan *taskqueue.Task)
	done := make(chan bool)
	i := 0
	u := routeUrl("update-feed")
	go taskSender(c, "update-feed", tc, done)
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		tc <- taskqueue.NewPOSTTask(u, url.Values{
			"feed": {k.StringID()},
		})
		i++
	}
	close(tc)
	<-done
	c.Infof("updating %d feeds", i)
}

func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(appengine.Timeout(c, time.Minute))
	url := r.FormValue("feed")
	if url == "" {
		c.Errorf("empty update feed")
		return
	}
	c.Debugf("update feed %s", url)
	last := len(r.FormValue("last")) > 0
	f := Feed{Url: url}
	s := ""
	defer func() {
		gn.Put(&Log{
			Parent: gn.Key(&f),
			Id:     time.Now().UnixNano(),
			Text:   "UpdateFeed - " + s,
		})
	}()
	if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
		c.Errorf("no such entity - " + url)
		s += "NSE"
		return
	} else if err != nil {
		s += "err - " + err.Error()
		return
	} else if last {
		// noop
	}
	if time.Now().Before(f.NextUpdate) {
		c.Errorf("feed %v already updated: %v", url, f.NextUpdate)
		s += "already updated"
		return
	}

	feedError := func(err error) {
		s += "feed err - " + err.Error()
		f.Errors++
		v := f.Errors + 1
		const max = 24 * 7
		if v > max {
			v = max
		} else if f.Errors == 1 {
			v = 0
		}
		f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v))
		gn.Put(&f)
		c.Warningf("error with %v (%v), bump next update to %v, %v", url, f.Errors, f.NextUpdate, err)
	}

	if feed, stories, err := fetchFeed(c, f.Url, f.Url); err == nil {
		if err := updateFeed(c, f.Url, feed, stories, false, false, last); err != nil {
			feedError(err)
		} else {
			s += "success"
		}
	} else {
		feedError(err)
	}
	f.Subscribe(c)
}

func reportHandler(w http.ResponseWriter, r *http.Request) {
	if r.FormValue("date") == "" {
		var params = map[string]interface{}{
			"Yesterday": date.NowInPdt().AddDate(0, 0, -1),
		}
		if err := templates.ExecuteTemplate(w, "report-form", params); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		return
	}

	c := appengine.Timeout(appengine.NewContext(r), 60*time.Second) // Default has a 5s timeout

	s, e, _ := widget.FormValueDateRange(r)
	opt := ReportOptions{
		ClassB_OnePerFlight:       widget.FormValueCheckbox(r, "classb_oneperflight"),
		ClassB_LocalDataOnly:      widget.FormValueCheckbox(r, "classb_localdataonly"),
		Skimmer_AltitudeTolerance: widget.FormValueFloat64(w, r, "skimmer_altitude_tolerance"),
		Skimmer_MinDurationNM:     widget.FormValueFloat64(w, r, "skimmer_min_duration_nm"),
	}

	if fix := strings.ToUpper(r.FormValue("waypoint")); fix != "" {
		if _, exists := sfo.KFixes[fix]; !exists {
			http.Error(w, fmt.Sprintf("Waypoint '%s' not known", fix), http.StatusInternalServerError)
			return
		}
		opt.Waypoint = fix
	}

	reportWriter(c, w, r, s, e, opt, r.FormValue("reportname"), r.FormValue("resultformat"))
}

// Upgrade the set of complaints for each user.
func upgradeUserHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.Timeout(appengine.NewContext(r), 30*time.Second)
	cdb := ComplaintDB{C: c, Memcache: false}

	p := types.ComplainerProfile{}
	if err := p.Base64Decode(r.FormValue("profile")); err != nil {
		c.Errorf("upgradeUserHandler: decode failed: %v", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	// Get *all* the complaints for this person, unfiltered.
	var data = []types.Complaint{}
	q := datastore.
		NewQuery(kComplaintKind).
		Ancestor(cdb.emailToRootKey(p.EmailAddress)).
		Order("Timestamp")
	keys, err := q.GetAll(c, &data)
	if err != nil {
		c.Errorf("upgradeUserHandler/%s: GetAll failed: %v", p.EmailAddress, err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	nDeleted, nUpdated := 0, 0
	str := ""
	for i, complaint := range data {
		FixupComplaint(&complaint, keys[i]) // Put the key where cdb.* expect to find it

		deleteMe := i < len(data)-1 && ComplaintsAreEquivalent(data[i], data[i+1])
		noProfile := complaint.Profile.EmailAddress == ""
		//str += fmt.Sprintf(" [%03d] [DEL=%5v,PRO=%5v] %s\n", i, deleteMe, noProfile, complaint)

		if true {
			if deleteMe {
				if err := cdb.DeleteComplaints([]string{complaint.DatastoreKey}, p.EmailAddress); err != nil {
					c.Errorf("upgradeUserHandler/%s: deletecomplaints failed: %v", p.EmailAddress, err)
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				nDeleted++

			} else if noProfile {
				complaint.Profile = p
				if err := cdb.UpdateComplaint(complaint, p.EmailAddress); err != nil {
					c.Errorf("upgradeUserHandler/%s: updatecomplaints failed: %v", p.EmailAddress, err)
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
				nUpdated++
			}
		}
	}

	str += fmt.Sprintf(" *** upgraded {%s} [tot=%d, del=%d, upd=%d]\n",
		p.EmailAddress, len(data), nDeleted, nUpdated)
	c.Infof(" -- Upgrade for %s --\n%s", p.EmailAddress, str)
	w.Write([]byte(fmt.Sprintf("OK, upgraded %s\n", p.EmailAddress)))
}

func downloadHandler(w http.ResponseWriter, r *http.Request) {
	session := sessions.Get(r)
	if session.Values["email"] == nil {
		http.Error(w, "session was empty; no cookie ? is this browser in privacy mode ?",
			http.StatusInternalServerError)
		return
	}

	c := appengine.Timeout(appengine.NewContext(r), 60*time.Second)

	cdb := complaintdb.ComplaintDB{C: c}
	cap, err := cdb.GetAllByEmailAddress(session.Values["email"].(string), true)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	_ = cap

	filename := date.NowInPdt().Format("complaints-20060102.csv")
	w.Header().Set("Content-Type", "application/csv")
	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))

	cols := []string{
		"Date", "Time(PDT)", "Notes", "Speedbrakes", "Loudness", "Activity",
		"Flightnumber", "Origin", "Destination", "Speed(Knots)", "Altitude(Feet)",
		"Lat", "Long", "Registration", "Callsign",
		"VerticalSpeed(FeetPerMin)", "Dist2(km)", "Dist3(km)",
	}

	csvWriter := csv.NewWriter(w)
	csvWriter.Write(cols)

	for _, c := range cap.Complaints {
		a := c.AircraftOverhead
		speedbrakes := ""
		if c.HeardSpeedbreaks {
			speedbrakes = "y"
		}
		r := []string{
			c.Timestamp.Format("2006/01/02"),
			c.Timestamp.Format("15:04:05"),
			c.Description, speedbrakes, fmt.Sprintf("%d", c.Loudness), c.Activity,
			a.FlightNumber, a.Origin, a.Destination,
			fmt.Sprintf("%.0f", a.Speed), fmt.Sprintf("%.0f", a.Altitude),
			fmt.Sprintf("%.5f", a.Lat), fmt.Sprintf("%.5f", a.Long),
			a.Registration, a.Callsign, fmt.Sprintf("%.0f", a.VerticalSpeed),
			fmt.Sprintf("%.1f", c.Dist2KM), fmt.Sprintf("%.1f", c.Dist3KM),
		}

		if err := csvWriter.Write(r); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	}
	csvWriter.Flush()
}

func NewDriver(ctx appengine.Context, kind string, logger wcg.Logger) *Driver {
	return &Driver{
		ctx:                    appengine.Timeout(ctx, DefaultTimeout),
		kind:                   kind,
		Logger:                 logger,
		EnableOperationLogging: DataStoreConfig.EnableOperationLogging,
		Timeout:                DefaultTimeout,
		namespace:              "",
		PutSplitThreshold:      DefaultPutSplitTreshold,
	}
}

func cannedSerfr1ComplaintsHandler(w http.ResponseWriter, r *http.Request) {
	s, e := date.WindowForYesterday()
	e = e.Add(-1 * time.Second)

	format := "list"
	if r.FormValue("csv") != "" {
		format = "csv"
	}

	c := appengine.Timeout(appengine.NewContext(r), 60*time.Second) // Default has a 5s timeout
	reportWriter(c, w, r, s, e, ReportOptions{}, "serfr1complaints", format)
}

func cannedAdsbClassBHandler(w http.ResponseWriter, r *http.Request) {
	s, e := date.WindowForYesterday()
	e = e.Add(-1 * time.Second)

	opt := ReportOptions{}
	format := "list"
	if r.FormValue("csv") != "" {
		format = "csv"
	}

	c := appengine.Timeout(appengine.NewContext(r), 60*time.Second) // Default has a 5s timeout
	reportWriter(c, w, r, s, e, opt, "adsbclassb", format)
}

func CronFetchHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.Timeout(appengine.NewContext(r), 30*time.Minute)

	q := datastore.NewQuery("Queue").KeysOnly()
	keys, _ := q.GetAll(c, nil)
	db := DB{c}
	start := time.Now()

	for _, v := range keys {
		db.FetchUser(strconv.FormatInt(v.IntID(), 10))
	}

	finish := time.Since(start)

	c.Infof("Finished cron fetch, took %s", finish)

	fmt.Fprint(w, "fetching users")
}

func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now())
	q = q.Limit(10 * 60 * 20) // 10/s queue, 20 min cron
	var keys []*datastore.Key
	it := q.Run(appengine.Timeout(c, time.Second*60))
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		keys = append(keys, k)
	}

	if len(keys) == 0 {
		c.Errorf("no results")
		return
	}
	c.Infof("updating %d feeds", len(keys))

	var tasks []*taskqueue.Task
	for _, k := range keys {
		tasks = append(tasks, taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		}))
	}
	var ts []*taskqueue.Task
	const taskLimit = 100
	for len(tasks) > 0 {
		if len(tasks) > taskLimit {
			ts = tasks[:taskLimit]
			tasks = tasks[taskLimit:]
		} else {
			ts = tasks
			tasks = tasks[0:0]
		}
		if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
}

func runTestTQ(
	c appengine.Context,
	r *http.Request,
) {
	//u.Debugf(c, "payload: %q", len(payload))
	c = appengine.Timeout(c, deadline)

	_, err := taskqueueAdd(c, &taskqueue.Task{
		Payload: payload,
		Method:  "PULL",
	}, "pull")
	if err != nil {
		u.Errorf(c, "%v", err)
		return
	}

	delayTestTQ.Call(c, "")
	return
}

func (g *Goon) putMemcache(srcs []interface{}, exists []byte) error {
	items := make([]*memcache.Item, len(srcs))
	payloadSize := 0
	for i, src := range srcs {
		toSerialize := src
		if exists[i] == 0 {
			toSerialize = nil
		}
		data, err := serializeStruct(toSerialize)
		if err != nil {
			g.error(err)
			return err
		}
		key, _, err := g.getStructKey(src)
		if err != nil {
			return err
		}
		// payloadSize will overflow if we push 2+ gigs on a 32bit machine
		payloadSize += len(data)
		items[i] = &memcache.Item{
			Key:   memkey(key),
			Value: data,
		}
	}
	memcacheTimeout := MemcachePutTimeoutSmall
	if payloadSize >= MemcachePutTimeoutThreshold {
		memcacheTimeout = MemcachePutTimeoutLarge
	}
	errc := make(chan error)
	go func() {
		errc <- memcache.SetMulti(appengine.Timeout(g.Context, memcacheTimeout), items)
	}()
	g.putMemoryMulti(srcs, exists)
	err := <-errc
	if appengine.IsTimeoutError(err) {
		g.timeoutError(err)
		err = nil
	} else if err != nil {
		g.error(err)
	}
	return err
}

func DeleteOldFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	ctx := appengine.Timeout(c, time.Minute)
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Kind(&Feed{})).Filter("n=", timeMax).KeysOnly()
	if cur, err := datastore.DecodeCursor(r.FormValue("c")); err == nil {
		q = q.Start(cur)
	}
	it := q.Run(ctx)
	done := false
	var tasks []*taskqueue.Task
	for i := 0; i < 10000 && len(tasks) < 100; i++ {
		k, err := it.Next(nil)
		if err == datastore.Done {
			c.Criticalf("done")
			done = true
			break
		} else if err != nil {
			c.Errorf("err: %v", err)
			continue
		}
		values := make(url.Values)
		values.Add("f", k.StringID())
		tasks = append(tasks, taskqueue.NewPOSTTask("/tasks/delete-old-feed", values))
	}
	if len(tasks) > 0 {
		c.Errorf("deleting %v feeds", len(tasks))
		if _, err := taskqueue.AddMulti(c, tasks, ""); err != nil {
			c.Errorf("err: %v", err)
		}
	}
	if !done {
		if cur, err := it.Cursor(); err == nil {
			values := make(url.Values)
			values.Add("c", cur.String())
			taskqueue.Add(c, taskqueue.NewPOSTTask("/tasks/delete-old-feeds", values), "")
		} else {
			c.Errorf("err: %v", err)
		}
	}
}

func DeleteOldFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	ctx := appengine.Timeout(c, time.Minute)
	g := goon.FromContext(ctx)
	oldDate := time.Now().Add(-time.Hour * 24 * 90)
	feed := Feed{Url: r.FormValue("f")}
	if err := g.Get(&feed); err != nil {
		c.Criticalf("err: %v", err)
		return
	}
	if feed.LastViewed.After(oldDate) {
		return
	}
	q := datastore.NewQuery(g.Kind(&Story{})).Ancestor(g.Key(&feed)).KeysOnly()
	keys, err := q.GetAll(ctx, nil)
	if err != nil {
		c.Criticalf("err: %v", err)
		return
	}
	q = datastore.NewQuery(g.Kind(&StoryContent{})).Ancestor(g.Key(&feed)).KeysOnly()
	sckeys, err := q.GetAll(ctx, nil)
	if err != nil {
		c.Criticalf("err: %v", err)
		return
	}
	keys = append(keys, sckeys...)
	c.Infof("delete: %v - %v", feed.Url, len(keys))
	feed.NextUpdate = timeMax.Add(time.Hour)
	if _, err := g.Put(&feed); err != nil {
		c.Criticalf("put err: %v", err)
	}
	if len(keys) == 0 {
		return
	}
	err = g.DeleteMulti(keys)
	if err != nil {
		c.Criticalf("err: %v", err)
	}
}

func DeleteBlobs(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	ctx := appengine.Timeout(c, time.Minute)
	q := datastore.NewQuery("__BlobInfo__").KeysOnly()
	it := q.Run(ctx)
	wg := sync.WaitGroup{}
	something := false
	for _i := 0; _i < 20; _i++ {
		var bk []appengine.BlobKey
		for i := 0; i < 1000; i++ {
			k, err := it.Next(nil)
			if err == datastore.Done {
				break
			} else if err != nil {
				c.Errorf("err: %v", err)
				continue
			}
			bk = append(bk, appengine.BlobKey(k.StringID()))
		}
		if len(bk) == 0 {
			break
		}
		go func(bk []appengine.BlobKey) {
			something = true
			c.Errorf("deleteing %v blobs", len(bk))
			err := blobstore.DeleteMulti(ctx, bk)
			if err != nil {
				c.Errorf("blobstore delete err: %v", err)
			}
			wg.Done()
		}(bk)
		wg.Add(1)
	}
	wg.Wait()
	if something {
		taskqueue.Add(c, taskqueue.NewPOSTTask("/tasks/delete-blobs", nil), "")
	}
}