本文整理汇总了Golang中appengine/taskqueue.AddMulti函数的典型用法代码示例。如果您正苦于以下问题:Golang AddMulti函数的具体用法?Golang AddMulti怎么用?Golang AddMulti使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了AddMulti函数的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Golang代码示例。
示例1: UpdateFeeds
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
q = q.Filter("n <=", time.Now())
q = q.Limit(1000)
it := gn.Run(q)
var keys []*datastore.Key
var del []*datastore.Key
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
} else if len(k.StringID()) == 0 {
del = append(del, k)
continue
}
keys = append(keys, k)
}
tasks := make([]*taskqueue.Task, len(keys))
for i, k := range keys {
tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
}
var ts []*taskqueue.Task
const taskLimit = 100
for len(tasks) > 0 {
if len(tasks) > taskLimit {
ts = tasks[:taskLimit]
tasks = tasks[taskLimit:]
} else {
ts = tasks
tasks = tasks[0:0]
}
if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
c.Infof("updating %d feeds", len(keys))
if len(del) > 0 {
c.Errorf("attempt to delete %v feeds", len(del))
if err := gn.DeleteMulti(del); err != nil {
c.Errorf("delete error: %v", err.Error())
}
fmt.Fprintf(w, `<html><head><meta http-equiv="refresh" content="0"></head></html>`)
fmt.Fprintf(w, "attempt to delete %v feeds", len(del))
for _, k := range del {
fmt.Fprintf(w, "\n<br>%v", k)
}
}
fmt.Fprintf(w, "updating %d feeds", len(keys))
}
示例2: taskHandler
func taskHandler(w http.ResponseWriter, r *http.Request) {
max := 12
var tasks []*taskqueue.Task
for i := 0; i < max; i++ {
tasks = append(tasks, taskqueue.NewPOSTTask("/", nil))
}
c := appengine.NewContext(r)
taskqueue.AddMulti(c, tasks, "read")
}
示例3: TestTasks
func TestTasks(t *testing.T) {
c, err := NewContext(&Options{TaskQueues: []string{"testQueue"}})
if err != nil {
t.Fatalf("NewContext: %v", err)
}
defer c.Close()
task := taskqueue.NewPOSTTask("/post", map[string][]string{})
_, err = taskqueue.Add(c, task, "testQueue")
if err != nil {
t.Fatalf("Could not add task to queue")
}
stats, err := taskqueue.QueueStats(c, []string{"testQueue"}, 0) // fetch all of them
if err != nil {
t.Fatalf("Could not get taskqueue statistics")
}
t.Logf("TaskStatistics = %#v", stats)
if len(stats) == 0 {
t.Fatalf("Queue statistics are empty")
} else if stats[0].Tasks != 1 {
t.Fatalf("Could not find the task we just added")
}
err = taskqueue.Purge(c, "testQueue")
if err != nil {
t.Fatalf("Could not purge the queue")
}
stats, err = taskqueue.QueueStats(c, []string{"testQueue"}, 0) // fetch all of them
if len(stats) == 0 {
t.Fatalf("Queue statistics are empty")
}
if stats[0].Tasks != 0 {
t.Fatalf("Purge command not successful")
}
tasks := []*taskqueue.Task{
taskqueue.NewPOSTTask("/post1", map[string][]string{}),
taskqueue.NewPOSTTask("/post2", map[string][]string{}),
}
_, err = taskqueue.AddMulti(c, tasks, "testQueue")
if err != nil {
t.Fatalf("Could not add bulk tasklist to queue")
}
stats, err = taskqueue.QueueStats(c, []string{"testQueue"}, 0) // fetch all of them
if err != nil {
t.Fatalf("Could not get taskqueue statistics")
}
if len(stats) == 0 {
t.Fatalf("Could not find the tasks we just added")
} else if stats[0].Tasks != 2 {
t.Fatalf("Could not find the tasks we just added")
}
}
示例4: addTasks
// addTasks adds the provided tasks in batches of 100 or less.
// This is to sidestep a limitation in the taskqueue API.
func addTasks(c appengine.Context, tasks []*taskqueue.Task, queue string) error {
n := 100
for len(tasks) > 0 {
if len(tasks) < n {
n = len(tasks)
}
_, err := taskqueue.AddMulti(c, tasks[:n], queue)
if err != nil {
return err
}
tasks = tasks[n:]
}
return nil
}
示例5: submitBulkUpdateUnique
func submitBulkUpdateUnique(c appengine.Context, w http.ResponseWriter, r *http.Request) {
r.ParseForm()
times := 1
tstr := r.FormValue("times")
if tstr != "" {
var err error
times, err = strconv.Atoi(tstr)
if err != nil {
times = 1
}
}
header := http.Header{
"Content-Type": []string{"application/x-www-form-urlencoded"},
}
tasks := []*taskqueue.Task{}
for ; times > 0; times-- {
tasks = append(tasks, &taskqueue.Task{
Path: "/batch/bulkUpdateUnique",
Payload: []byte(r.Form.Encode()),
Header: header,
})
if len(tasks) >= 100 {
_, err := taskqueue.AddMulti(c, tasks, "bulkupdateunique")
c.Infof("Submitted %v tasks: %v", len(tasks), err)
tasks = nil
}
}
if len(tasks) >= 0 {
_, err := taskqueue.AddMulti(c, tasks, "bulkupdateunique")
c.Infof("Submitted %v tasks (final): %v", len(tasks), err)
}
w.WriteHeader(202)
}
示例6: UpdateFeeds
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
q := datastore.NewQuery("F").KeysOnly()
q = q.Filter("n <=", time.Now())
q = q.Limit(3000)
var keys []*datastore.Key
it := q.Run(c)
retry := 0
for {
k, err := it.Next(nil)
if err != nil {
c.Errorf("next error: %v, retry: %v", err.Error(), retry)
if retry == 5 {
break
}
retry++
it = q.Run(c)
} else {
keys = append(keys, k)
}
}
if len(keys) == 0 {
c.Errorf("giving up")
return
}
tasks := make([]*taskqueue.Task, len(keys))
for i, k := range keys {
c.Infof("task: %v, %v", i, k)
tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
}
var ts []*taskqueue.Task
const taskLimit = 100
for len(tasks) > 0 {
if len(tasks) > taskLimit {
ts = tasks[:taskLimit]
tasks = tasks[taskLimit:]
} else {
ts = tasks
tasks = tasks[0:0]
}
if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
c.Infof("updating %d feeds", len(keys))
}
示例7: UpdateFeeds
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
q := datastore.NewQuery("F").KeysOnly().Filter("n <=", time.Now())
q = q.Limit(10 * 60 * 20) // 10/s queue, 20 min cron
var keys []*datastore.Key
it := q.Run(appengine.Timeout(c, time.Second*60))
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
}
keys = append(keys, k)
}
if len(keys) == 0 {
c.Errorf("no results")
return
}
c.Infof("updating %d feeds", len(keys))
var tasks []*taskqueue.Task
for _, k := range keys {
tasks = append(tasks, taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
}))
}
var ts []*taskqueue.Task
const taskLimit = 100
for len(tasks) > 0 {
if len(tasks) > taskLimit {
ts = tasks[:taskLimit]
tasks = tasks[taskLimit:]
} else {
ts = tasks
tasks = tasks[0:0]
}
if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
}
示例8: taskSender
func taskSender(c mpg.Context, queue string, tc chan *taskqueue.Task, done chan bool) {
const taskLimit = 100
tasks := make([]*taskqueue.Task, 0, taskLimit)
send := func() {
taskqueue.AddMulti(c, tasks, queue)
c.Infof("added %v tasks", len(tasks))
tasks = tasks[0:0]
}
for t := range tc {
tasks = append(tasks, t)
if len(tasks) == taskLimit {
send()
}
}
if len(tasks) > 0 {
send()
}
done <- true
}
示例9: UpdateFeeds
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
gn := goon.FromContext(c)
q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
q = q.Filter("n <=", time.Now())
q = q.Limit(2500)
it := gn.Run(q)
var keys []*datastore.Key
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
}
keys = append(keys, k)
}
tasks := make([]*taskqueue.Task, len(keys))
for i, k := range keys {
tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
}
var ts []*taskqueue.Task
const taskLimit = 100
for len(tasks) > 0 {
if len(tasks) > taskLimit {
ts = tasks[:taskLimit]
tasks = tasks[taskLimit:]
} else {
ts = tasks
tasks = tasks[0:0]
}
if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
c.Infof("updating %d feeds", len(keys))
fmt.Fprintf(w, "updating %d feeds", len(keys))
}
示例10: UpdateFeeds
func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
q := datastore.NewQuery("F").KeysOnly()
q = q.Filter("n <=", time.Now())
q = q.Limit(3000)
var keys []*datastore.Key
for i := 0; i < 5; i++ {
if _keys, err := q.GetAll(c, nil); err != nil {
c.Errorf("get all error: %v, retry %v", err.Error(), i)
} else {
c.Errorf("got %v keys", len(_keys))
keys = _keys
break
}
}
if len(keys) == 0 {
c.Errorf("giving up")
return
}
tasks := make([]*taskqueue.Task, len(keys))
for i, k := range keys {
tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
"feed": {k.StringID()},
})
}
var ts []*taskqueue.Task
const taskLimit = 100
for len(tasks) > 0 {
if len(tasks) > taskLimit {
ts = tasks[:taskLimit]
tasks = tasks[taskLimit:]
} else {
ts = tasks
tasks = tasks[0:0]
}
if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
c.Infof("updating %d feeds", len(keys))
}
示例11: RSVPMeetupEvents
func RSVPMeetupEvents(events []MeetupEvent, r *http.Request) string {
c := appengine.NewContext(r)
queuetasks := make([]*taskqueue.Task, 0)
taskqueue.Purge(c, "futurersvp")
for _, event := range events {
t := ProcessEvent(event, r)
if t != nil {
queuetasks = append(queuetasks, t)
}
}
taskqueue.AddMulti(c, queuetasks, "futurersvp")
str := fmt.Sprintf("Total Number of Meetups processed: %d\n", len(events))
str = str + fmt.Sprintf("Total Number of future Meetups queued: %d\n", len(queuetasks))
log.Print(str)
return str
}
示例12: load
func load(w http.ResponseWriter, r *http.Request, c appengine.Context, output *crBuffer) int {
// Determine how many tasks to queue
count := 10
countStr := r.URL.Query().Get("count")
if countStr != "" {
parsedCount, err := strconv.ParseInt(countStr, 10, 64)
if err != nil {
output.WriteLine("Could not parse count: %s", countStr)
return http.StatusBadRequest
}
if parsedCount <= 0 {
output.WriteLine("Invalid count specified: %d", parsedCount)
return http.StatusBadRequest
}
count = int(parsedCount)
}
delayStr := r.URL.Query().Get("delay")
if delayStr == "" {
delayStr = "10"
}
tasks := make([]*taskqueue.Task, count)
for i := 0; i < count; i++ {
tasks[i] = taskqueue.NewPOSTTask(
"/work",
url.Values{
"delay": []string{delayStr},
})
}
_, err := taskqueue.AddMulti(c, tasks, "auto-worker-push")
if err != nil {
c.Errorf("Error loading tasks: %s", err)
output.WriteLine("Error loading tasks")
return http.StatusInternalServerError
}
output.WriteLine("Loaded %d tasks into queue with delay %s", count, delayStr)
return http.StatusOK
}
示例13: DeleteOldFeeds
func DeleteOldFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
ctx := appengine.Timeout(c, time.Minute)
gn := goon.FromContext(c)
q := datastore.NewQuery(gn.Kind(&Feed{})).Filter("n=", timeMax).KeysOnly()
if cur, err := datastore.DecodeCursor(r.FormValue("c")); err == nil {
q = q.Start(cur)
}
it := q.Run(ctx)
done := false
var tasks []*taskqueue.Task
for i := 0; i < 10000 && len(tasks) < 100; i++ {
k, err := it.Next(nil)
if err == datastore.Done {
c.Criticalf("done")
done = true
break
} else if err != nil {
c.Errorf("err: %v", err)
continue
}
values := make(url.Values)
values.Add("f", k.StringID())
tasks = append(tasks, taskqueue.NewPOSTTask("/tasks/delete-old-feed", values))
}
if len(tasks) > 0 {
c.Errorf("deleting %v feeds", len(tasks))
if _, err := taskqueue.AddMulti(c, tasks, ""); err != nil {
c.Errorf("err: %v", err)
}
}
if !done {
if cur, err := it.Cursor(); err == nil {
values := make(url.Values)
values.Add("c", cur.String())
taskqueue.Add(c, taskqueue.NewPOSTTask("/tasks/delete-old-feeds", values), "")
} else {
c.Errorf("err: %v", err)
}
}
}
示例14: batchPrepareUnique
func batchPrepareUnique(c appengine.Context, w http.ResponseWriter, r *http.Request) {
keyStr := []string{}
z, err := gzip.NewReader(r.Body)
maybePanic(err)
d := json.NewDecoder(z)
maybePanic(d.Decode(&keyStr))
c.Infof("Got %v keys to pull", len(keyStr))
keys := []*datastore.Key{}
obs := make([]oldLoader, len(keyStr))
for i, k := range keyStr {
key, err := datastore.DecodeKey(k)
if err != nil {
c.Errorf("Error decoding key: %v: %v", k, err)
http.Error(w, err.Error(), 500)
return
}
keys = append(keys, key)
obs[i] = oldLoader{c: c, into: &Stats{}}
}
err = datastore.GetMulti(c, keys, obs)
if err != nil {
c.Errorf("Error grabbing the things: %v", err)
http.Error(w, err.Error(), 500)
return
}
tasks := []*taskqueue.Task{}
for i, ol := range obs {
s := ol.into.(*Stats)
s.uncompress()
uuid, err := getUUID(*s)
if err != nil {
c.Errorf("Couldn't find UUID from %v: %v", keyStr[i], err)
}
if uuid != "" {
tasks = append(tasks, &taskqueue.Task{
Method: "PULL",
Tag: uuid,
Payload: []byte(keyStr[i]),
})
}
}
_, err = taskqueue.AddMulti(c, tasks, "updateUniquePull")
c.Infof("Response for adding %v pull tasks: %v", len(tasks), err)
if err != nil {
c.Errorf("Error queueing tasks: %v", err)
http.Error(w, err.Error(), 500)
return
}
c.Infof("Queued %v tasks", len(tasks))
// Try to submit a bulk updater. This can cause a chain
// reaction. It'll be awesome
taskqueue.Add(c, taskqueue.NewPOSTTask("/batch/bulkUpdateUnique", url.Values{}),
"bulkupdateunique")
w.WriteHeader(204)
}
示例15: CFixer
func CFixer(c mpg.Context, w http.ResponseWriter, r *http.Request) {
q := datastore.NewQuery("F").KeysOnly()
q = q.Limit(1000)
cs := r.FormValue("c")
if len(cs) > 0 {
if cur, err := datastore.DecodeCursor(cs); err == nil {
q = q.Start(cur)
c.Infof("starting at %v", cur)
} else {
c.Errorf("cursor error %v", err.Error())
}
}
var keys []*datastore.Key
it := q.Run(Timeout(c, time.Second*15))
for {
k, err := it.Next(nil)
if err == datastore.Done {
break
} else if err != nil {
c.Errorf("next error: %v", err.Error())
break
}
keys = append(keys, k)
}
if len(keys) == 0 {
c.Errorf("no results")
return
} else {
cur, err := it.Cursor()
if err != nil {
c.Errorf("to cur error %v", err.Error())
} else {
c.Infof("add with cur %v", cur)
t := taskqueue.NewPOSTTask("/tasks/cfixer", url.Values{
"c": {cur.String()},
})
taskqueue.Add(c, t, "cfixer")
}
}
c.Infof("fixing %d feeds", len(keys))
var tasks []*taskqueue.Task
for _, k := range keys {
c.Infof("f: %v", k.StringID())
tasks = append(tasks, taskqueue.NewPOSTTask("/tasks/cfix", url.Values{
"feed": {k.StringID()},
}))
}
var ts []*taskqueue.Task
const taskLimit = 100
for len(tasks) > 0 {
if len(tasks) > taskLimit {
ts = tasks[:taskLimit]
tasks = tasks[taskLimit:]
} else {
ts = tasks
tasks = tasks[0:0]
}
if _, err := taskqueue.AddMulti(c, ts, "cfixer"); err != nil {
c.Errorf("taskqueue error: %v", err.Error())
}
}
}