Golang datastore.GetMulti函数代码示例

// ByIDs filters out Tasks that are not visible to the current User.
func (s *TaskService) ByIDs(ctx context.Context, ids []string) ([]*Task, error) {
	span := trace.FromContext(ctx).NewChild("trythings.task.ByIDs")
	defer span.Finish()

	rootKey := datastore.NewKey(ctx, "Root", "root", 0, nil)

	ks := []*datastore.Key{}
	for _, id := range ids {
		ks = append(ks, datastore.NewKey(ctx, "Task", id, 0, rootKey))
	}

	var allTasks = make([]*Task, len(ks))
	err := datastore.GetMulti(ctx, ks, allTasks)
	if err != nil {
		return nil, err
	}

	ts := []*Task{}
	for _, t := range allTasks {
		// TODO#Perf: Batch the isVisible check.
		ok, err := s.IsVisible(ctx, t)
		if err != nil {
			return nil, err
		}
		if !ok {
			continue
		}

		ts = append(ts, t)
	}

	return ts, nil
}

// 2015-11-06 to force field EditSummary (even if empty) on every IdiomHistory persisted entity.
func resaveAllIdiomHistory(c context.Context) error {
	defer memcache.Flush(c)
	saved := 0
	keys, err := datastore.NewQuery("IdiomHistory").KeysOnly().GetAll(c, nil)
	if err != nil {
		return err
	}
	nbEntities := len(keys)

	defer func() {
		log.Infof(c, "Resaved %d IdiomHistory entities out of %d.", saved, nbEntities)
	}()

	for len(keys) > 0 {
		bunch := 100
		if len(keys) < bunch {
			bunch = len(keys)
		}
		histories := make([]*IdiomHistory, bunch)
		err := datastore.GetMulti(c, keys[:bunch], histories)
		if err != nil {
			return err
		}
		_, err = datastore.PutMulti(c, keys[:bunch], histories)
		if err != nil {
			return err
		}
		saved += bunch

		// Remove processed keys
		keys = keys[bunch:]
	}
	return nil
}

// GetMulti wraps datastore.GetMulti
func (d *Driver) GetMulti(key []*datastore.Key, dst interface{}) error {
	var keyLen = len(key)
	var fromIdx, toIdx int
	var v = reflect.ValueOf(dst)
	// TODO: split multiple goroutine
	for {
		fromIdx = toIdx
		toIdx = fromIdx + 999
		if toIdx > keyLen {
			toIdx = keyLen
		}
		_keys := key[fromIdx:toIdx]
		_data := v.Slice(fromIdx, toIdx).Interface()
		d.logOps(opRead, len(_keys), "GetMulti")
		if err := datastore.GetMulti(d.ctx, _keys, _data); err != nil {
			return err
		}
		v1 := reflect.ValueOf(_data)
		for i := 0; i < toIdx-fromIdx; i++ {
			v.Index(fromIdx + i).Set(v1.Index(i))
		}
		if toIdx == keyLen {
			break
		}
	}
	return nil
}

// filterNextSessions queries kindNext entities and returns a subset of items
// containing only the elements not present in the datastore, previously saved with
// storeNextSessions().
func filterNextSessions(c context.Context, items []*eventSession) ([]*eventSession, error) {
	pkey := nextSessionParent(c)
	keys := make([]*datastore.Key, len(items))
	for i, s := range items {
		id := s.ID + ":" + s.Update
		keys[i] = datastore.NewKey(c, kindNext, id, 0, pkey)
	}
	zeros := make([]struct{}, len(keys))
	err := datastore.GetMulti(c, keys, zeros)
	merr, ok := err.(appengine.MultiError)
	if !ok && err != nil {
		return nil, err
	}
	res := make([]*eventSession, 0, len(keys))
	for i, e := range merr {
		if e == nil {
			continue
		}
		if e != datastore.ErrNoSuchEntity {
			return nil, err
		}
		res = append(res, items[i])
	}
	return res, nil
}

func averageResulter(ctx context.Context, result model.KeyedResult, challenge model.Challenge) error {
	tasks := make([]model.Task, len(challenge.Tasks))
	if err := datastore.GetMulti(ctx, challenge.Tasks, tasks); err != nil {
		return err
	}

	var user model.User
	if err := datastore.Get(ctx, result.Key.Parent().Parent(), &user); err != nil {
		return err
	}

	var nrOfComputations float64
	average := model.Skills{}

	for i, task := range tasks {
		taskResult, err := Tasker(task.Tasker).Call(ctx, result, *task.Key(challenge.Tasks[i]), user, result.StartTimes[getTaskIndex(challenge, challenge.Tasks[i])])
		if err != nil {
			// TODO: ignore error for now. We`ll treat it after we have all the taskers available
			//return err
		} else {
			average = average.Add(taskResult)
			nrOfComputations++
		}
	}

	result.Skills = average.DivBy(nrOfComputations)
	result.Computed = time.Now()

	_, err := result.Put(ctx, result.Key)
	return err
}

func (s *Store) LoadMagazines(keys []*datastore.Key, a []*Magazine) error {
	if err := datastore.GetMulti(s.c, keys, a); err != nil {
		return err
	}
	for _, m := range a {
		m.Init(s.c)
	}
	return nil
}

func (s *Store) LoadUsers(keys []*datastore.Key, a []*User) error {
	if err := datastore.GetMulti(s.c, keys, a); err != nil {
		return err
	}
	for _, u := range a {
		u.Init(s.c)
	}
	return nil
}

func (d rdsImpl) GetMulti(keys []ds.Key, _meta ds.MultiMetaGetter, cb ds.GetMultiCB) error {
	rkeys := dsMF2R(keys)
	vals := make([]datastore.PropertyLoadSaver, len(keys))
	for i := range keys {
		vals[i] = &typeFilter{ds.PropertyMap{}}
	}
	err := datastore.GetMulti(d, rkeys, vals)
	return idxCallbacker(err, len(keys), func(idx int, err error) {
		cb(vals[idx].(*typeFilter).pm, err)
	})
}

func example() {
	// [START batch]
	// A batch put.
	_, err = datastore.PutMulti(ctx, []*datastore.Key{k1, k2, k3}, []interface{}{e1, e2, e3})

	// A batch get.
	var entities = make([]*T, 3)
	err = datastore.GetMulti(ctx, []*datastore.Key{k1, k2, k3}, entities)

	// A batch delete.
	err = datastore.DeleteMulti(ctx, []*datastore.Key{k1, k2, k3})
	// [END batch]
	_ = err
}

func GetChallengesForProfile(ctx context.Context, w http.ResponseWriter, r *http.Request) (status int, err error) {
	if r.Method != "GET" {
		return http.StatusMethodNotAllowed, nil
	}

	_, ok := passenger.FromContext(ctx)
	if !ok {
		return http.StatusUnauthorized, nil
	}

	var profileKey *datastore.Key
	if profileKey, err = datastore.DecodeKey(mux.Vars(r)["key"]); err != nil {
		return http.StatusInternalServerError, err
	}

	q := model.NewQueryForResult().
		Ancestor(profileKey)
	if finished := r.URL.Query()["finished"]; len(finished) > 0 && finished[0] == "true" {
		q = q.Filter("Finished >", time.Time{})
	}
	if order := r.URL.Query()["order"]; len(order) > 0 && order[0] != "" {
		q = q.Order(order[0])
	}

	if limitQuery := r.URL.Query()["limit"]; len(limitQuery) > 0 {
		if limit, err := strconv.Atoi(limitQuery[0]); err != nil {
			return http.StatusInternalServerError, err
		} else {
			q = q.Limit(limit)
		}
	}

	var results model.Results
	if _, err = q.GetAll(ctx, &results); err != nil {
		return http.StatusInternalServerError, err
	}

	challengeKeys := make([]*datastore.Key, len(results))
	for i, val := range results {
		challengeKeys[i] = val.Challenge
	}

	challenges := make(model.Challenges, len(challengeKeys))
	if err = datastore.GetMulti(ctx, challengeKeys, challenges); err != nil {
		return http.StatusInternalServerError, err
	}

	json.NewEncoder(w).Encode(challenges.Key(challengeKeys))
	return
}

func (d rdsImpl) GetMulti(keys []*ds.Key, _meta ds.MultiMetaGetter, cb ds.GetMultiCB) error {
	vals := make([]datastore.PropertyLoadSaver, len(keys))
	rkeys, err := dsMF2R(d.aeCtx, keys)
	if err == nil {
		for i := range keys {
			vals[i] = &typeFilter{d.aeCtx, ds.PropertyMap{}}
		}
		err = datastore.GetMulti(d.aeCtx, rkeys, vals)
	}
	return idxCallbacker(err, len(keys), func(idx int, err error) {
		if pls := vals[idx]; pls != nil {
			cb(pls.(*typeFilter).pm, err)
		} else {
			cb(nil, err)
		}
	})
}

// Load reads the JSON representation of entities from the io.Reader "r",
// and stores them in the Datastore using the given context.Context.
// The Options parameter allows you to configure how the dump will work.
// If there is any parsing erros, improper format, or datastore failures
// during the process, that error is returned and processing stops. The
// error may be returned after some entities were loaded: there is no
// parsing cache.
func Load(c context.Context, r io.Reader, o *Options) error {
	entities, err := DecodeEntities(c, r)
	if err != nil {
		return err
	}
	if len(entities) == 0 {
		log.Infof(c, "Skipping load of 0 entities")
		return nil
	}
	batchSize := o.BatchSize
	if batchSize <= 0 {
		batchSize = 50
	}
	for start, end := 0, 0; start < len(entities); {
		end += batchSize
		if end > len(entities) {
			end = len(entities)
		}
		keys := make([]*datastore.Key, 0, end-start)
		values := make([]datastore.PropertyList, 0, cap(keys))

		for _, e := range entities[start:end] {
			keys = append(keys, e.Key)
			values = append(values, e.Properties)
		}

		keys, err = datastore.PutMulti(c, keys, values)
		if err != nil {
			return err
		}
		log.Infof(c, "Loaded %d entities ...", len(keys))

		if o.GetAfterPut {
			log.Infof(c, "Making a read to force consistency ...")
			l := make([]Entity, len(keys))
			err := datastore.GetMulti(c, keys, l)
			if err != nil {
				return err
			}
		}

		start = end
	}

	return nil
}

func adminMarkPaid(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	action := r.FormValue("action")

	keys := make([]*datastore.Key, 0, len(r.Form["pay"]))
	for _, s := range r.Form["pay"] {
		k, err := datastore.DecodeKey(s)
		if err != nil {
			panic(err)
		}
		keys = append(keys, k)
	}

	if action == "Mark Paid" {
		tasks := make([]LoggedTask, len(keys))

		err := datastore.GetMulti(c, keys, tasks)
		if err != nil {
			panic(err)
		}

		now := time.Now().UTC()
		for i := range tasks {
			tasks[i].Paid = true
			tasks[i].PaidTime = now
		}

		_, err = datastore.PutMulti(c, keys, tasks)
		if err != nil {
			panic(err)
		}
	} else if action == "Delete" {
		err := datastore.DeleteMulti(c, keys)
		if err != nil {
			panic(err)
		}
	} else {
		panic("Unhandled action: " + action)
	}

	http.Redirect(w, r, "/admin/", 303)
}

func executeIdiomTextSearchQuery(c context.Context, query string, limit int) ([]*Idiom, error) {
	// log.Infof(c, query)
	index, err := gaesearch.Open("idioms")
	if err != nil {
		return nil, err
	}
	if limit == 0 {
		// Limit is not optional. 0 means zero result.
		return nil, nil
	}
	idiomKeys := make([]*datastore.Key, 0, limit)
	// This is an *IDsOnly* search, where docID == Idiom.Id
	it := index.Search(c, query, &gaesearch.SearchOptions{
		Limit:   limit,
		IDsOnly: true,
	})
	for {
		docID, err := it.Next(nil)
		if err == gaesearch.Done {
			break
		}
		if err != nil {
			return nil, err
		}
		idiomID, err := strconv.Atoi(docID)
		if err != nil {
			return nil, err
		}
		key := newIdiomKey(c, idiomID)
		idiomKeys = append(idiomKeys, key)
	}
	// Fetch Idioms in a []Idiom
	buffer := make([]Idiom, len(idiomKeys))
	err = datastore.GetMulti(c, idiomKeys, buffer)
	// Convert []Idiom to []*Idiom
	idioms := make([]*Idiom, len(buffer))
	for i := range buffer {
		// Do not take the address of the 2nd range variable, it would make a copy.
		// Better take the address in the existing buffer.
		idioms[i] = &buffer[i]
	}
	return idioms, err
}

func (iter *LongBatchingIterator) NextWithErr() (*types.Complaint, error) {
	if iter.err != nil {
		return nil, iter.err
	}

	if len(iter.vals) == 0 && len(iter.keys) == 0 {
		return nil, nil // We're all done !
	}

	// No new vals left in the cache; fetch some
	if len(iter.vals) == 0 {
		var keysForThisBatch []*datastore.Key

		if len(iter.keys) < iter.BatchSize {
			// Remaining keys not enough for a full page; grab all of 'em
			keysForThisBatch = iter.keys
			iter.keys = []*datastore.Key{}
		} else {
			keysForThisBatch = iter.keys[0:iter.BatchSize]
			iter.keys = iter.keys[iter.BatchSize:]
		}

		// Fetch the complaints for the keys in this batch
		complaints := make([]types.Complaint, len(keysForThisBatch))
		if err := datastore.GetMulti(iter.Ctx, keysForThisBatch, complaints); err != nil {
			iter.err = err
			return nil, err
		}

		iter.vals = make([]*types.Complaint, len(keysForThisBatch))
		for i, _ := range complaints {
			FixupComplaint(&complaints[i], keysForThisBatch[i])
			iter.vals[i] = &complaints[i]
		}
	}

	// We have unreturned results in the cache; shift & return it
	complaint := iter.vals[0]
	iter.vals = iter.vals[1:]

	return complaint, nil
}