Golang datastore.NewQuery函数代码示例

// CleanupDatastore is to remove all data in datastore
func CleanupDatastore(ctx context.Context, namespaces ...string) error {
	var dummy []interface{}
	logger := wcg.NewLogger(nil)
	logger.Debugf("[Fixture] --------- CleanupDatastore ---------")
	namespaceList := append(namespaces, "")
	for _, ns := range namespaceList {
		logger.Debugf("[Fixture] Cleanup: ns=%q", ns)
		var _ctx = ctx
		if ns != "" {
			_ctx, _ = appengine.Namespace(_ctx, ns)
		}
		err := wcg.RetryUntil(func() error {
			var keys []*datastore.Key
			var err error
			if keys, err = datastore.NewQuery("").KeysOnly().GetAll(_ctx, dummy); err != nil {
				return err
			}
			if err := datastore.DeleteMulti(_ctx, keys); err != nil {
				return err
			}
			count, _ := datastore.NewQuery("").KeysOnly().Count(_ctx)
			if count == 0 {
				return nil
			}
			return fmt.Errorf("Still have %d keys.", count)
		}, 10*time.Second, 100*time.Millisecond)
		if err != nil {
			return err
		}
	}
	return nil
}

func cleanup(s *testerator.Setup) error {
	t := datastore.NewQuery("__kind__").KeysOnly().Run(s.Context)
	kinds := make([]string, 0)
	for {
		key, err := t.Next(nil)
		if err == datastore.Done {
			break
		}
		if err != nil {
			return err
		}
		kinds = append(kinds, key.StringID())
	}

	for _, kind := range kinds {
		q := datastore.NewQuery(kind).KeysOnly()
		keys, err := q.GetAll(s.Context, nil)
		if err != nil {
			return err
		}
		err = datastore.DeleteMulti(s.Context, keys)
		if err != nil {
			return err
		}
	}

	return nil
}

// Dump exports entities from the context c using the specified Options o and
// writing the generated JSON representations to the io.Writer w. You can configure
// how the dump will run by using the Options parameter. If there is an error
// generating the output, or writting to the writer, it is returned. This method
// may return an error after writting bytes to w: the output is not buffered.
func Dump(c context.Context, w io.Writer, o *Options) error {
	var (
		comma        = []byte(",")
		openBracket  = []byte("[")
		closeBracket = []byte("]")
		lineFeed     = []byte("\n")
		indent       = "  "
	)

	w.Write(openBracket)
	count := 0
	last := 0
	batchSize := o.BatchSize
	if batchSize <= 0 {
		batchSize = 100
	}

	q := datastore.NewQuery(o.Kind).Limit(batchSize)
	for i := q.Run(c); ; {
		var e Entity
		k, err := i.Next(&e)
		e.Key = k
		if err == datastore.Done {
			log.Infof(c, "datastore.Done: last=%d, count=%d", last, count)
			if last == count || count-last < batchSize {
				break
			}
			// This 100 batch is done, but more can be found in the next one
			last = count
			cur, err := i.Cursor()
			if err != nil {
				return err
			}
			log.Infof(c, "restarting the query: cursor=%v", cur)
			i = datastore.NewQuery(o.Kind).Limit(batchSize).Start(cur).Run(c)
			continue
		}
		if err != nil {
			return err
		}
		if count > 0 {
			w.Write(comma)
			w.Write(lineFeed)
		}
		var b []byte
		if o.PrettyPrint {
			b, err = json.MarshalIndent(&e, "", indent)
		} else {
			b, err = json.Marshal(&e)
		}
		if err != nil {
			return err
		}
		w.Write(b)
		count++
	}
	w.Write(closeBracket)
	return nil
}

// DeleteAll deletes across all roots
// DeleteAll deletes by kind alone.
func (fs *dsFileSys) DeleteAll() (string, error) {

	msg := ""
	{
		q := datastore.NewQuery(tfil).KeysOnly()
		var files []DsFile
		keys, err := q.GetAll(fs.Ctx(), &files)
		if err != nil {
			msg += "could not get file keys\n"
			return msg, err
		}
		if len(keys) >= 500 {
			msg += "limited to 500 files. REPEAT operation.\n"
			keys = keys[:500]
		}

		err = datastore.DeleteMulti(fs.Ctx(), keys)
		if err != nil {
			msg += "error deleting files\n"
			return msg, err
		}

		msg += spf("%v files deleted\n", len(keys))

	}

	{
		q := datastore.NewQuery(tdir).KeysOnly()
		var dirs []DsDir
		keys, err := q.GetAll(fs.Ctx(), &dirs)
		if err != nil {
			msg += "could not get dir keys\n"
			return msg, err
		}
		if len(keys) >= 500 {
			msg += "limited to 500 directories. REPEAT operation.\n"
			keys = keys[:500]
		}

		err = datastore.DeleteMulti(fs.Ctx(), keys)
		if err != nil {
			msg += "error deleting directories\n"
			return msg, err
		}

		msg += spf("%v directories deleted\n", len(keys))
	}

	err := memcache.Flush(fs.Ctx())
	if err != nil {
		msg += "error flushing memcache\n"
		return msg, err
	} else {
		msg += "memcache flushed \n"
	}

	return msg, nil
}

// subtreeByPath retrieves a subdirectories of a given directory.
// It is relying on an indexed string property "Dir"
// containing a string representation of the full path.
//
// It might be fast for deep, uncached directory subtree,
// that have been saved in nested manner.
//
// However, it might not find recently added directories.
// Upon finding nothing, it therefore returns the
// "warning" fsi.EmptyQueryResult
//
// The func could easily be enhanced chunked scanning.
//
// It is currently used by ReadDir and by the test package.
// It is public for the test package.
func (fs *dsFileSys) SubtreeByPath(name string, onlyDirectChildren bool) ([]DsDir, error) {

	dir, bname := fs.SplitX(name)
	name = dir + common.Filify(bname)
	if !strings.HasSuffix(name, sep) {
		name += sep
	}

	var q *datastore.Query

	if onlyDirectChildren {
		q = datastore.NewQuery(tdir).
			Filter("Dir=", name).
			Order("Dir")
		//  Limit(4)
	} else {
		pathInc := IncrementString(name)
		q = datastore.NewQuery(tdir).
			Filter("Dir>=", name).
			Filter("Dir<", pathInc).
			Order("Dir")
	}

	// log.Printf("%v", q)

	var children []DsDir
	keys, err := q.GetAll(fs.Ctx(), &children)
	if err != nil {
		aelog.Errorf(fs.Ctx(), "Error getting all children of %v => %v", dir+bname, err)
		return children, err
	}

	if len(children) < 1 {
		return children, fsi.EmptyQueryResult
	}

	// Very evil: We filter out root node, since it's
	// has the same dir as the level-1 directories.
	keyRoot := datastore.NewKey(fs.Ctx(), tdir, fs.RootDir(), 0, nil)
	idxRoot := -1

	for i := 0; i < len(children); i++ {
		children[i].fSys = fs
		children[i].Key = keys[i]
		if keys[i].Equal(keyRoot) {
			// log.Printf("root idx %v", i)
			idxRoot = i
		}
	}

	if idxRoot > -1 {
		children = append(children[:idxRoot], children[idxRoot+1:]...)
	}

	return children, nil

}

func example4() {
	// [START sort_order_example]
	// Order alphabetically by last name:
	q := datastore.NewQuery("Person").Order("LastName")

	// Order by height, tallest to shortest:
	q = datastore.NewQuery("Person").Order("-Height")
	// [END sort_order_example]
	_ = q
}

func GetTestResultsForSubmission(ctx context.Context, w http.ResponseWriter, r *http.Request) (status int, err error) {
	if r.Method != "GET" {
		return http.StatusMethodNotAllowed, nil
	}

	// TODO(victorbalan): Check if user is company or if user is parent of result
	// else return http.StatusUnauthorized
	_, ok := passenger.FromContext(ctx)
	if !ok {
		return http.StatusUnauthorized, nil
	}

	submissionKey, err := datastore.DecodeKey(mux.Vars(r)["key"])
	if err != nil {
		return http.StatusNotFound, err
	}

	keys, err := datastore.NewQuery("").
		Ancestor(submissionKey).
		Filter("__key__ >", submissionKey).
		KeysOnly().
		GetAll(ctx, nil)
	if err != nil {
		return http.StatusInternalServerError, err
	}
	if len(keys) == 0 {
		json.NewEncoder(w).Encode([]string{})
		return http.StatusOK, nil
	}

	switch keys[0].Kind() {
	case model.JunitTestResultKind:
		var results model.JunitTestResults
		_, err = datastore.NewQuery(keys[0].Kind()).
			Ancestor(submissionKey).
			GetAll(ctx, &results)
		if err != nil {
			return http.StatusInternalServerError, err
		}
		json.NewEncoder(w).Encode(results)
	case model.DiffTestResultKind:
		var results model.DiffTestResults
		_, err = datastore.NewQuery(keys[0].Kind()).
			Ancestor(submissionKey).
			GetAll(ctx, &results)
		if err != nil {
			return http.StatusInternalServerError, err
		}
		json.NewEncoder(w).Encode(results)
	default:
		w.Write([]byte("[]"))
	}
	return http.StatusOK, nil
}

func getRelations(req *http.Request, username string) ([]Relation, []Relation, error) {
	ctx := appengine.NewContext(req)
	var Following, FollowingMe []Relation
	q := datastore.NewQuery("Relation")
	_, err := q.Filter("Follower =", username).Order("Following").GetAll(ctx, &Following)
	if err != nil {
		return []Relation{}, []Relation{}, err
	}
	q = datastore.NewQuery("Relation")
	_, err = q.Filter("Following =", username).Order("Follower").GetAll(ctx, &FollowingMe)
	if err != nil {
		return []Relation{}, []Relation{}, err
	}
	return Following, FollowingMe, nil
}

func profile(res http.ResponseWriter, req *http.Request, ps httprouter.Params) {
	sd := sessionInfo(req)
	var user User
	user.UserName = ps.ByName("name")

	ctx := appengine.NewContext(req)

	//Get ppl they're following
	q := datastore.NewQuery("Follow").Filter("Follower =", user.UserName)
	var f []Follow
	_, err := q.GetAll(ctx, &f)
	if err != nil {
		panic(err)
	}
	for _, val := range f {
		user.Following = append(user.Following, val.Following)
	}

	//Get ppl they're followed by
	q2 := datastore.NewQuery("Follow").Filter("Following =", user.UserName)
	var f2 []Follow
	_, err2 := q2.GetAll(ctx, &f2)
	if err2 != nil {
		panic(err)
	}
	for _, val := range f2 {
		user.FollowedBy = append(user.FollowedBy, val.Follower)
	}

	//Get ppl they're following
	q3 := datastore.NewQuery("Follow").Filter("Follower =", sd.UserName)
	var f3 []Follow
	_, err = q3.GetAll(ctx, &f3)
	if err != nil {
		panic(err)
	}
	for _, val := range f3 {
		sd.Following = append(sd.Following, val.Following)
	}

	sd.FollowingUser = stringInSlice(user.UserName, sd.Following)
	if user.UserName == sd.UserName {
		sd.FollowingUser = true
	}
	sd.ViewingUser = user

	tpl.ExecuteTemplate(res, "profile.html", &sd)
}

func projections(c context.Context, u User, days int) (int64, int64, error) {
	var projected, earned int64

	g := syncutil.Group{}

	g.Go(func() error {
		q := datastore.NewQuery("Task").
			Filter("Disabled = ", false).
			Filter("Assignee = ", u.Email)

		for t := q.Run(c); ; {
			var x Task
			_, err := t.Next(&x)
			if err == datastore.Done {
				return nil
			} else if err != nil {
				return err
			}

			log.Debugf(c, "Item worth %v every %v", x.Value, x.Period)

			projected += int64(float64(x.Value) * (float64(days) / float64(x.Period)))
		}
	})

	g.Go(func() error {
		q := datastore.NewQuery("LoggedTask").
			Filter("User = ", u.Key).
			Filter("Completed >=", time.Now().Add(-24*time.Hour*time.Duration(days)))

		for t := q.Run(c); ; {
			var x LoggedTask
			_, err := t.Next(&x)
			if err == datastore.Done {
				return nil
			} else if err != nil {
				return err
			}

			log.Debugf(c, "Logged task worth %v", x.Amount)

			earned += int64(x.Amount)
		}
	})
	g.Wait()

	return projected, earned, g.Err()
}

func getWord(res http.ResponseWriter, req *http.Request) {
	ctx := appengine.NewContext(req)
	term := req.FormValue("term")

	q := datastore.NewQuery("Word").Order("Term").Filter("Term =", term)

	html := " "

	iterator := q.Run(ctx)

	for {
		var entity Word
		_, err := iterator.Next(&entity)
		if err == datastore.Done {
			break
		} else if err != nil {
			http.Error(res, err.Error(), 500)
			return
		}
		html += `<dt>` + entity.Term + `</dt>
				 <dd>` + entity.Def + `</dd>`
	}

	res.Header().Set("Content-Type", "text/html")
	fmt.Fprintln(res, `
	<dl>`+html+`<dl>
	<form method="POST" action="">
		Enter a term to see its definition <input type="text" name="term">
		<input type="submit">
	</form>
		`)
}

func (s *TestSuite) TestAddLocation() {
	t := s.T()

	loc, err := json.Marshal(&Location{
		Name: "Cambridge Fresh Pond",
		Lat:  42.5,
		Lng:  -71.5,
	})
	assert.Nil(t, err, "Error marshalling Location into JSON: %v", err)

	req, err := s.inst.NewRequest("POST", "/add-location",
		ioutil.NopCloser(bytes.NewBuffer(loc)))
	require.Nil(t, err, "Error preparing request: %v", err)
	rec := httptest.NewRecorder()
	s.rt.ServeHTTP(rec, req)

	const expectedResponse = `{"addLocation":"success"}`
	assert.Equal(t, expectedResponse, string(rec.Body.Bytes()),
		"Expected response to be %s, got %s", expectedResponse, string(rec.Body.Bytes()))

	dbReq, err := s.inst.NewRequest("GET", "/", nil)
	require.Nil(t, err, "Error preparing request: %v", err)
	ctx := appengine.NewContext(dbReq)

	q := datastore.NewQuery("Location")
	numLocs, err := q.Count(ctx)
	require.Nil(t, err, "Error preparing request: %v", err)
	assert.Equal(t, 1, numLocs, "Expected number of locations to be 1, got %d", numLocs)
}

func TestAddLocationTestify(t *testing.T) {
	inst, err := aetest.NewInstance(
		&aetest.Options{StronglyConsistentDatastore: true})
	require.Nil(t, err, "Error creating aetest instance: %v", err)
	defer inst.Close()
	rt := initRouter()

	loc, err := json.Marshal(&Location{
		Name: "Cambridge Fresh Pond",
		Lat:  42.5,
		Lng:  -71.5,
	})
	assert.Nil(t, err, "Error marshalling Location into JSON: %v", err)

	req, err := inst.NewRequest("POST", "/add-location",
		ioutil.NopCloser(bytes.NewBuffer(loc)))
	require.Nil(t, err, "Error preparing request: %v", err)
	rec := httptest.NewRecorder()
	rt.ServeHTTP(rec, req)

	const expectedResponse = `{"addLocation":"success"}`
	assert.Equal(t, expectedResponse, string(rec.Body.Bytes()),
		"Expected response to be %s, got %s", expectedResponse, string(rec.Body.Bytes()))

	dbReq, err := inst.NewRequest("GET", "/", nil)
	require.Nil(t, err, "Error preparing request: %v", err)
	ctx := appengine.NewContext(dbReq)

	q := datastore.NewQuery("Location")
	numLocs, err := q.Count(ctx)
	require.Nil(t, err, "Error preparing request: %v", err)
	assert.Equal(t, 1, numLocs, "Expected number of locations to be 1, got %d", numLocs)
}

func handlePersonalPage(res http.ResponseWriter, req *http.Request) {
	ctx := appengine.NewContext(req)
	session := getSession(ctx, req)
	username := strings.SplitN(req.URL.Path, "/", 3)[2]
	if username == "" {
		http.Redirect(res, req, "/userslist", 302)
		return
	}
	q := datastore.NewQuery("Image").Filter("Username =", username)
	var images []Image
	q.GetAll(ctx, &images)
	key := datastore.NewKey(ctx, "List", username, 0, nil)
	var list List
	datastore.Get(ctx, key, &list)
	listItems := make([]string, 0)
	if strings.TrimSpace(list.List) != "" {
		listItems = strings.Split(list.List, "\n")
	}
	model := &personalModel{
		Session:   session,
		Username:  username,
		ListItems: listItems,
		Images:    images,
	}
	err := tpls.ExecuteTemplate(res, "personalpage", model)
	if err != nil {
		http.Error(res, err.Error(), 500)
		return
	}
}

func (cdb ComplaintDB) LoadGlobalStats() (*GlobalStats, error) {
	gs := GlobalStats{}

	fgs := []FrozenGlobalStats{}
	q := datastore.NewQuery(kGlobalStatsKind).Limit(10)

	if keys, err := q.GetAll(cdb.Ctx(), &fgs); err != nil {
		return nil, err
	} else if len(fgs) != 1 {
		return nil, fmt.Errorf("LoadGlobalStats: found %d, expected 1", len(fgs))
	} else {
		buf := bytes.NewBuffer(fgs[0].Bytes)
		err := gob.NewDecoder(buf).Decode(&gs)
		gs.DatastoreKey = keys[0].Encode() // Store this, so we can overwrite

		// Pick out the high water marks ...
		if len(gs.Counts) > 0 {
			iMaxComplaints, iMaxComplainers := 0, 0
			maxComplaints, maxComplainers := 0, 0
			for i, _ := range gs.Counts {
				if gs.Counts[i].NumComplaints > maxComplaints {
					iMaxComplaints, maxComplaints = i, gs.Counts[i].NumComplaints
				}
				if gs.Counts[i].NumComplainers > maxComplainers {
					iMaxComplainers, maxComplainers = i, gs.Counts[i].NumComplainers
				}
			}
			gs.Counts[iMaxComplaints].IsMaxComplaints = true
			gs.Counts[iMaxComplainers].IsMaxComplainers = true
		}

		return &gs, err
	}
}