Golang parser.ParseQuery函数代码示例

func (self *FilteringSuite) TestReturnAllColumnsIfAskedForWildcard(c *C) {
	queryStr := "select * from t where column_one == 100 and column_two != 6;"
	query, err := parser.ParseQuery(queryStr)
	c.Assert(err, IsNil)
	series, err := common.StringToSeriesArray(`
[
 {
   "points": [
     {"values": [{"int64_value": 100},{"int64_value": 5 }], "timestamp": 1381346631, "sequence_number": 1},
     {"values": [{"int64_value": 100},{"int64_value": 6 }], "timestamp": 1381346631, "sequence_number": 1},
     {"values": [{"int64_value": 90 },{"int64_value": 15}], "timestamp": 1381346632, "sequence_number": 1}
   ],
   "name": "t",
   "fields": ["column_one", "column_two"]
 }
]
`)
	c.Assert(err, IsNil)
	result, err := Filter(query, series[0])
	c.Assert(err, IsNil)
	c.Assert(result, NotNil)
	c.Assert(result.Points, HasLen, 1)
	c.Assert(result.Fields, HasLen, 2)
	c.Assert(result.Points[0].Values, HasLen, 2)
}

func (self *DatastoreSuite) TestPropagateErrorsProperly(c *C) {
	cleanup(nil)
	db := newDatastore(c)
	defer cleanup(db)
	mock := `
  {
    "points": [
      {
        "values": [
          {
            "int64_value": 3
          }
        ],
        "sequence_number": 1
      }
    ],
    "name": "foo",
    "fields": ["value"]
  }`
	pointTime := time.Now().Unix()
	series := stringToSeries(mock, pointTime, c)
	err := db.WriteSeriesData("test", series)
	c.Assert(err, IsNil)
	q, err := parser.ParseQuery("select value from foo;")
	c.Assert(err, IsNil)
	yield := func(series *protocol.Series) error {
		return fmt.Errorf("Whatever")
	}
	user := &MockUser{}
	err = db.ExecuteQuery(user, "test", q, yield)
	c.Assert(err, ErrorMatches, "Whatever")
}

func (self *FilteringSuite) TestInequalityFiltering(c *C) {
	queryStr := "select * from t where column_one >= 100 and column_two > 6 and time > now() - 1d;"
	query, err := parser.ParseQuery(queryStr)
	c.Assert(err, IsNil)
	series, err := common.StringToSeriesArray(`
[
 {
   "points": [
     {"values": [{"int64_value": 100},{"int64_value": 7 }], "timestamp": 1381346631, "sequence_number": 1},
     {"values": [{"int64_value": 100},{"int64_value": 6 }], "timestamp": 1381346631, "sequence_number": 1},
     {"values": [{"int64_value": 90 },{"int64_value": 15}], "timestamp": 1381346632, "sequence_number": 1}
   ],
   "name": "t",
   "fields": ["column_one", "column_two"]
 }
]
`)
	c.Assert(err, IsNil)
	result, err := Filter(query, series[0])
	c.Assert(err, IsNil)
	c.Assert(result, NotNil)
	c.Assert(result.Points, HasLen, 1)
	c.Assert(*result.Points[0].Values[0].Int64Value, Equals, int64(100))
	c.Assert(*result.Points[0].Values[1].Int64Value, Equals, int64(7))
}

func (self *QueryEngine) RunQuery(user common.User, database string, query string, yield func(*protocol.Series) error) (err error) {
	// don't let a panic pass beyond RunQuery
	defer func() {
		if err := recover(); err != nil {
			fmt.Fprintf(os.Stderr, "********************************BUG********************************\n")
			buf := make([]byte, 1024)
			n := runtime.Stack(buf, false)
			fmt.Fprintf(os.Stderr, "Database: %s\n", database)
			fmt.Fprintf(os.Stderr, "Query: [%s]\n", query)
			fmt.Fprintf(os.Stderr, "Error: %s. Stacktrace: %s\n", err, string(buf[:n]))
			err = common.NewQueryError(common.InternalError, "Internal Error")
		}
	}()

	q, err := parser.ParseQuery(query)
	if err != nil {
		return err
	}

	if isAggregateQuery(q) {
		return self.executeCountQueryWithGroupBy(user, database, q, yield)
	} else {
		return self.distributeQuery(user, database, q, yield)
	}
	return nil
}

func (self *CoordinatorSuite) TestShouldQuerySequentially(c *C) {
	end := time.Now().Truncate(24 * time.Hour)
	start := end.Add(-7 * 24 * time.Hour)
	shard := cluster.NewShard(1, start, end, cluster.SHORT_TERM, false, nil)
	shards := []*cluster.ShardData{shard}
	coordinator := NewCoordinatorImpl(&configuration.Configuration{
		ClusterMaxResponseBufferSize: 1000,
	}, nil, nil)
	queries := map[string]bool{
		"list series": false,
		"select count(foo) from /.*bar.*/ group by time(1d)": true,
		"select count(foo) from bar":                         true,
		"select foo from bar":                                true,
		"select count(foo) from bar group by baz":            true,
		"select count(foo) from bar group by time(1d)":       false,
		"select count(foo) from bar group by time(3d)":       true,
	}

	for query, result := range queries {
		fmt.Printf("Testing %s\n", query)
		parsedQuery, err := parser.ParseQuery(query)
		c.Assert(err, IsNil)
		c.Assert(parsedQuery, HasLen, 1)
		querySpec := parser.NewQuerySpec(nil, "", parsedQuery[0])
		c.Assert(coordinator.shouldQuerySequentially(shards, querySpec), Equals, result)
	}
}

func (self *QueryEngine) RunQuery(user common.User, database string, queryString string, yield func(*protocol.Series) error) (err error) {
	// don't let a panic pass beyond RunQuery
	defer recoverFunc(database, queryString)

	q, err := parser.ParseQuery(queryString)
	if err != nil {
		return err
	}

	for _, query := range q {
		if query.DeleteQuery != nil {
			if err := self.coordinator.DeleteSeriesData(user, database, query.DeleteQuery); err != nil {
				return err
			}
			continue
		}

		selectQuery := query.SelectQuery
		if isAggregateQuery(selectQuery) {
			return self.executeCountQueryWithGroupBy(user, database, selectQuery, yield)
		} else if containsArithmeticOperators(selectQuery) {
			return self.executeArithmeticQuery(user, database, selectQuery, yield)
		} else {
			return self.distributeQuery(user, database, selectQuery, yield)
		}
	}
	return nil
}

func (self *DatastoreSuite) TestCanWriteAndRetrievePoints(c *C) {
	cleanup(nil)
	db := newDatastore(c)
	defer cleanup(db)
	mock := `
  {
    "points": [
      {
        "values": [
          {
            "int64_value": 3
          }
        ],
        "sequence_number": 1
      },
      {
        "values": [
          {
            "int64_value": 2
          }
        ],
        "sequence_number": 2
      }
    ],
    "name": "foo",
    "fields": ["value"]
  }`
	pointTime := time.Now().Unix()
	series := stringToSeries(mock, pointTime, c)
	err := db.WriteSeriesData("test", series)
	c.Assert(err, IsNil)
	q, errQ := parser.ParseQuery("select value from foo;")
	c.Assert(errQ, IsNil)
	resultSeries := []*protocol.Series{}
	yield := func(series *protocol.Series) error {
		resultSeries = append(resultSeries, series)
		return nil
	}
	user := &MockUser{}
	err = db.ExecuteQuery(user, "test", q, yield)
	c.Assert(err, IsNil)
	// we should get the actual data and the end of series data
	// indicator , i.e. a series with no points
	c.Assert(resultSeries, HasLen, 2)
	c.Assert(resultSeries[0].Points, HasLen, 2)
	c.Assert(resultSeries[0].Fields, HasLen, 1)
	c.Assert(*resultSeries[0].Points[0].SequenceNumber, Equals, uint32(2))
	c.Assert(*resultSeries[0].Points[1].SequenceNumber, Equals, uint32(1))
	c.Assert(*resultSeries[0].Points[0].GetTimestampInMicroseconds(), Equals, pointTime*1000000)
	c.Assert(*resultSeries[0].Points[1].GetTimestampInMicroseconds(), Equals, pointTime*1000000)
	c.Assert(*resultSeries[0].Points[0].Values[0].Int64Value, Equals, int64(2))
	c.Assert(*resultSeries[0].Points[1].Values[0].Int64Value, Equals, int64(3))
	c.Assert(resultSeries[1].Points, HasLen, 0)
	c.Assert(resultSeries[1].Fields, HasLen, 1)
	c.Assert(resultSeries, Not(DeepEquals), series)
}

func (self *DatastoreSuite) TestCanSelectFromARegex(c *C) {
	cleanup(nil)
	db := newDatastore(c)
	defer cleanup(db)

	mock := `{
    "points":[
      {"values":[{"int64_value":3},{"string_value":"paul"}],"sequence_number":2},
      {"values":[{"int64_value":1},{"string_value":"todd"}],"sequence_number":1}],
      "name":"user_things",
      "fields":["count", "name"]
    }`
	series := stringToSeries(mock, time.Now().Unix(), c)
	err := db.WriteSeriesData("foobar", series)
	c.Assert(err, IsNil)
	user := &MockUser{}
	results := executeQuery(user, "foobar", "select count, name from user_things;", db, c)
	c.Assert(results[0], DeepEquals, series)

	mock = `{
    "points":[{"values":[{"double_value":10.1}],"sequence_number":23}],
    "name":"response_times",
    "fields":["ms"]
  }`
	responseSeries := stringToSeries(mock, time.Now().Unix(), c)
	err = db.WriteSeriesData("foobar", responseSeries)
	c.Assert(err, IsNil)
	results = executeQuery(user, "foobar", "select ms from response_times;", db, c)
	c.Assert(results[0], DeepEquals, responseSeries)

	mock = `{
    "points":[{"values":[{"string_value":"NY"}],"sequence_number":23}, {"values":[{"string_value":"CO"}],"sequence_number":20}],
    "name":"other_things",
    "fields":["state"]
  }`
	otherSeries := stringToSeries(mock, time.Now().Unix(), c)
	err = db.WriteSeriesData("foobar", otherSeries)
	c.Assert(err, IsNil)
	results = executeQuery(user, "foobar", "select state from other_things;", db, c)
	c.Assert(results[0], DeepEquals, otherSeries)

	q, errQ := parser.ParseQuery("select * from /.*things/;")
	c.Assert(errQ, IsNil)
	resultSeries := make([]*protocol.Series, 0)
	yield := func(series *protocol.Series) error {
		if len(series.Points) > 0 {
			resultSeries = append(resultSeries, series)
		}
		return nil
	}
	err = db.ExecuteQuery(user, "foobar", q, yield)
	c.Assert(err, IsNil)
	c.Assert(resultSeries, HasLen, 2)
	c.Assert(resultSeries[0], DeepEquals, otherSeries)
	c.Assert(resultSeries[1], DeepEquals, series)
}

func (self *DatastoreSuite) TestCanDeleteRangeOfDataFromRegex(c *C) {
	cleanup(nil)
	db := newDatastore(c)
	defer cleanup(db)

	mock := `{
    "points":[
      {"values":[{"int64_value":3},{"string_value":"paul"}],"sequence_number":2},
      {"values":[{"int64_value":1},{"string_value":"todd"}],"sequence_number":1}
    ],
    "name":"events",
    "fields":["count","name"]
  }`
	series := stringToSeries(mock, time.Now().Unix(), c)
	err := db.WriteSeriesData("foobar", series)
	c.Assert(err, IsNil)
	user := &MockUser{}
	results := executeQuery(user, "foobar", "select count, name from events;", db, c)
	c.Assert(results[0], DeepEquals, series)

	mock = `{
    "points":[{"values":[{"double_value":10.1}],"sequence_number":23}],
    "name":"response_times",
    "fields":["ms"]
  }`
	responseSeries := stringToSeries(mock, time.Now().Unix(), c)
	err = db.WriteSeriesData("foobar", responseSeries)
	c.Assert(err, IsNil)
	results = executeQuery(user, "foobar", "select ms from response_times;", db, c)
	c.Assert(results[0], DeepEquals, responseSeries)

	mock = `{
    "points":[{"values":[{"double_value":232.1}],"sequence_number":23}, {"values":[{"double_value":10.1}],"sequence_number":20}],
    "name":"queue_time",
    "fields":["processed_time"]
  }`
	otherSeries := stringToSeries(mock, time.Now().Unix(), c)
	err = db.WriteSeriesData("foobar", otherSeries)
	c.Assert(err, IsNil)
	results = executeQuery(user, "foobar", "select processed_time from queue_time;", db, c)
	c.Assert(results[0], DeepEquals, otherSeries)

	queries, _ := parser.ParseQuery("delete from /.*time.*/ where time > now() - 1h")
	db.DeleteSeriesData("foobar", queries[0].DeleteQuery)

	results = executeQuery(user, "foobar", "select * from events;", db, c)
	c.Assert(results[0], DeepEquals, series)
	results = executeQuery(user, "foobar", "select * from response_times;", db, c)
	c.Assert(results, HasLen, 0)
	results = executeQuery(user, "foobar", "select * from queue_time;", db, c)
	c.Assert(results, HasLen, 0)
}

func (self *CoordinatorImpl) handleReplayRequest(r *protocol.Request, replicationFactor *uint8, owningServerId *uint32) {
	err := self.datastore.LogRequestAndAssignSequenceNumber(r, replicationFactor, owningServerId)
	if err != nil {
		log.Error("Error writing waiting requests after replay: %s", err)
	}
	if *r.Type == protocol.Request_PROXY_WRITE || *r.Type == protocol.Request_REPLICATION_WRITE {
		log.Debug("Replaying write request")
		self.datastore.WriteSeriesData(*r.Database, r.Series)
	} else if *r.Type == protocol.Request_PROXY_DELETE || *r.Type == protocol.Request_REPLICATION_DELETE {
		query, _ := parser.ParseQuery(*r.Query)
		err = self.datastore.DeleteSeriesData(*r.Database, query[0].DeleteQuery)
	}
}

func (self *DatastoreSuite) TestCanWriteAndRetrievePointsWithAlias(c *C) {
	cleanup(nil)
	db := newDatastore(c)
	defer cleanup(db)
	mock := `
  {
    "points": [
      {
        "values": [
          {
            "int64_value": 3
          }
        ],
        "sequence_number": 1
      },
      {
        "values": [
          {
            "int64_value": 2
          }
        ],
        "sequence_number": 2
      }
    ],
    "name": "foo",
    "fields": ["value"]
  }`
	pointTime := time.Now().Unix()
	series := stringToSeries(mock, pointTime, c)
	err := db.WriteSeriesData("test", series)
	c.Assert(err, IsNil)
	q, errQ := parser.ParseQuery("select * from foo as f1 inner join foo as f2;")
	c.Assert(errQ, IsNil)
	resultSeries := map[string][]*protocol.Series{}
	yield := func(series *protocol.Series) error {
		resultSeries[*series.Name] = append(resultSeries[*series.Name], series)
		return nil
	}
	user := &MockUser{}
	err = db.ExecuteQuery(user, "test", q, yield)
	c.Assert(err, IsNil)
	// we should get the actual data and the end of series data
	// indicator , i.e. a series with no points
	c.Assert(resultSeries, HasLen, 2)
	c.Assert(resultSeries["f1"], HasLen, 2)
	c.Assert(resultSeries["f1"][0].Points, HasLen, 2)
	c.Assert(resultSeries["f1"][1].Points, HasLen, 0)
	c.Assert(resultSeries["f2"], HasLen, 2)
	c.Assert(resultSeries["f2"][0].Points, HasLen, 2)
	c.Assert(resultSeries["f2"][1].Points, HasLen, 0)
}

func executeQuery(user common.User, database, query string, db Datastore, c *C) []*protocol.Series {
	q, errQ := parser.ParseQuery(query)
	c.Assert(errQ, IsNil)
	resultSeries := []*protocol.Series{}
	yield := func(series *protocol.Series) error {
		// ignore time series which have no data, this includes
		// end of series indicator
		if len(series.Points) > 0 {
			resultSeries = append(resultSeries, series)
		}
		return nil
	}
	err := db.ExecuteQuery(user, database, q, yield)
	c.Assert(err, IsNil)
	return resultSeries
}

func (self *DatastoreSuite) TestCheckWriteAccess(c *C) {
	cleanup(nil)
	db := newDatastore(c)
	defer cleanup(db)

	mock := `{
    "points":[
      {"values":[{"int64_value":3},{"string_value":"paul"}],"sequence_number":2},
      {"values":[{"int64_value":1},{"string_value":"todd"}],"sequence_number":1}],
      "name":"user_things",
      "fields":["count", "name"]
    }`
	series := stringToSeries(mock, time.Now().Unix(), c)
	err := db.WriteSeriesData("foobar", series)
	c.Assert(err, IsNil)

	mock = `{
    "points":[{"values":[{"string_value":"NY"}],"sequence_number":23}, {"values":[{"string_value":"CO"}],"sequence_number":20}],
    "name":"other_things",
    "fields":["state"]
  }`
	otherSeries := stringToSeries(mock, time.Now().Unix(), c)
	err = db.WriteSeriesData("foobar", otherSeries)
	c.Assert(err, IsNil)

	user := &MockUser{
		dbCannotWrite: map[string]bool{"other_things": true},
	}
	regex, _ := regexp.Compile(".*")
	err = db.DeleteRangeOfRegex(user, "foobar", regex, time.Now().Add(-time.Hour), time.Now())
	c.Assert(err, ErrorMatches, ".*one or more.*")

	q, errQ := parser.ParseQuery("select * from /.*things/;")
	c.Assert(errQ, IsNil)
	resultSeries := make([]*protocol.Series, 0)
	yield := func(series *protocol.Series) error {
		if len(series.Points) > 0 {
			resultSeries = append(resultSeries, series)
		}
		return nil
	}
	err = db.ExecuteQuery(user, "foobar", q, yield)
	c.Assert(err, IsNil)
	c.Assert(resultSeries, HasLen, 1)
	c.Assert(resultSeries[0], DeepEquals, otherSeries)
}

func (self *ProtobufRequestHandler) handleQuery(request *protocol.Request, conn net.Conn) {
	// the query should always parse correctly since it was parsed at the originating server.
	queries, err := parser.ParseQuery(*request.Query)
	if err != nil || len(queries) < 1 {
		log.Error("Error parsing query: ", err)
		errorMsg := fmt.Sprintf("Cannot find user %s", *request.UserName)
		response := &protocol.Response{Type: &endStreamResponse, ErrorMessage: &errorMsg, RequestId: request.Id}
		self.WriteResponse(conn, response)
		return
	}
	query := queries[0]
	var user common.User
	if *request.IsDbUser {
		user = self.clusterConfig.GetDbUser(*request.Database, *request.UserName)
	} else {
		user = self.clusterConfig.GetClusterAdmin(*request.UserName)
	}

	if user == nil {
		errorMsg := fmt.Sprintf("Cannot find user %s", *request.UserName)
		response := &protocol.Response{Type: &accessDeniedResponse, ErrorMessage: &errorMsg, RequestId: request.Id}
		self.WriteResponse(conn, response)
		return
	}

	shard := self.clusterConfig.GetLocalShardById(*request.ShardId)

	querySpec := parser.NewQuerySpec(user, *request.Database, query)

	responseChan := make(chan *protocol.Response)
	if querySpec.IsDestructiveQuery() {
		go shard.HandleDestructiveQuery(querySpec, request, responseChan, true)
	} else {
		go shard.Query(querySpec, responseChan)
	}
	for {
		response := <-responseChan
		response.RequestId = request.Id
		self.WriteResponse(conn, response)
		if response.GetType() == protocol.Response_END_STREAM || response.GetType() == protocol.Response_ACCESS_DENIED {
			return
		}
	}
}

func (self *DatastoreSuite) TestBreaksLargeResultsIntoMultipleBatches(c *C) {
	cleanup(nil)
	db := newDatastore(c)
	defer cleanup(db)

	mock := `{
    "points":[
      {"values":[{"double_value":23.1},{"string_value":"paul"}],"sequence_number":2},
      {"values":[{"double_value":56.8},{"string_value":"todd"}],"sequence_number":1}],
      "name":"user_things",
      "fields":["response_time","name"]
  }`
	series := stringToSeries(mock, time.Now().Unix(), c)
	sequence := 0
	writtenPoints := 0
	for i := 0; i < 50000; i++ {
		for _, p := range series.Points {
			sequence += 1
			s := uint32(sequence)
			p.SequenceNumber = &s
		}
		writtenPoints += 2
		err := db.WriteSeriesData("foobar", series)
		c.Assert(err, IsNil)
	}

	q, errQ := parser.ParseQuery("select * from user_things;")
	c.Assert(errQ, IsNil)
	resultSeries := make([]*protocol.Series, 0)
	yield := func(series *protocol.Series) error {
		resultSeries = append(resultSeries, series)
		return nil
	}
	user := &MockUser{}
	err := db.ExecuteQuery(user, "foobar", q, yield)
	c.Assert(err, IsNil)
	c.Assert(len(resultSeries), InRange, 2, 20)
	pointCount := 0
	for _, s := range resultSeries {
		pointCount += len(s.Points)
	}
	c.Assert(pointCount, Equals, writtenPoints)
}