Golang bson.MongoTimestamp函数代码示例

func (s *oplogSuite) TestRestartsOnError(c *gc.C) {
	_, session := s.startMongo(c)

	oplog := s.makeFakeOplog(c, session)
	tailer := mongo.NewOplogTailer(oplog, nil, time.Time{})
	defer tailer.Stop()

	// First, ensure that the tailer is seeing oplog inserts.
	s.insertDoc(c, session, oplog, &mongo.OplogDoc{
		Timestamp:   1,
		OperationId: 99,
	})
	doc := s.getNextOplog(c, tailer)
	c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))

	s.emptyCapped(c, oplog)

	// Ensure that the tailer still works.
	s.insertDoc(c, session, oplog, &mongo.OplogDoc{
		Timestamp:   2,
		OperationId: 42,
	})
	doc = s.getNextOplog(c, tailer)
	c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(2))
}

func TestTimestamp(t *testing.T) {
	data := []struct {
		in     M
		want   bson.MongoTimestamp
		wantok bool
	}{
		{
			map[string]interface{}{"$timestamp": map[string]interface{}{"t": 1392822881, "i": 1}},
			bson.MongoTimestamp(5982128723015499777),
			true,
		},
		{
			map[string]interface{}{"$ugh": map[string]interface{}{"t": 1392822881, "i": 1}},
			bson.MongoTimestamp(5982128723015499777),
			false,
		},
	}

	for _, d := range data {
		b, ok := d.in.timestamp()
		if ok != d.wantok {
			t.Errorf("got %t, want %t, (%v)", ok, d.wantok, d.in)
			t.FailNow()
		}
		if ok && b != d.want {
			t.Errorf("wanted: %v (%T), got: %v (%T)", d.want, d.want, b, b)
		}
	}
}

func TestValidOplogLimitChecking(t *testing.T) {

	testutil.VerifyTestType(t, testutil.UnitTestType)

	Convey("With a MongoRestore instance with oplogLimit of 5:0", t, func() {
		mr := &MongoRestore{
			oplogLimit: bson.MongoTimestamp(int64(5) << 32),
		}

		Convey("an oplog entry with ts=1000:0 should be invalid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(1000)<<32)), ShouldBeFalse)
		})

		Convey("an oplog entry with ts=5:1 should be invalid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(5)<<32|1)), ShouldBeFalse)
		})

		Convey("an oplog entry with ts=5:0 should be invalid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(5)<<32)), ShouldBeFalse)
		})

		Convey("an oplog entry with ts=4:9 should be valid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(4)<<32|9)), ShouldBeTrue)
		})

		Convey("an oplog entry with ts=4:0 should be valid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(4)<<32)), ShouldBeTrue)
		})

		Convey("an oplog entry with ts=0:1 should be valid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(1)), ShouldBeTrue)
		})
	})

	Convey("With a MongoRestore instance with no oplogLimit", t, func() {
		mr := &MongoRestore{}

		Convey("an oplog entry with ts=1000:0 should be valid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(1000)<<32)), ShouldBeTrue)
		})

		Convey("an oplog entry with ts=5:1 should be valid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(5)<<32|1)), ShouldBeTrue)
		})

		Convey("an oplog entry with ts=5:0 should be valid", func() {
			So(mr.TimestampBeforeLimit(bson.MongoTimestamp(int64(5)<<32)), ShouldBeTrue)
		})
	})

}

func OplogDeamon() {

	url := fmt.Sprintf("mongodb://%s:%s", Conf.MongoIP, Conf.MongoPort)
	session, err := mgo.Dial(url)
	if err != nil {
		panic(err)
	}
	defer session.Close()

	c := session.DB("local").C("oplog.rs")

	cp := initCheckpoint()

	log.Info("init check point is %d ", cp)

	iter := c.Find(bson.M{"ts": bson.M{"$gte": bson.MongoTimestamp(cp)}}).Tail(-1 * time.Second)
	defer iter.Close()

	count := uint64(0)
	for {

		log.Info("start oplog manarger !!! ")

		for result := new(Oplog); iter.Next(&result); result = new(Oplog) {

			count++
			checkpoint_threshold(count, result)

			if !checkTopicCollection(result.Ns) {
				log.Info("the namespace(%s) not in topics: %s", result.Ns, strings.Join(Conf.TopicCollections, "|"))
				continue
			}

			// new event
			newEvent(result)

		}

		if iter.Err() != nil {
			iter.Close()
		}
		if iter.Timeout() {
			continue
		}

		// reload checkp point from file
		cp = initCheckpoint()
		iter = c.Find(bson.M{"ts": bson.M{"$gte": bson.MongoTimestamp(cp)}}).Tail(-1 * time.Second)
	}

}

func (s *oplogSuite) TestNoRepeatsAfterIterRestart(c *gc.C) {
	// A bunch of documents with the same timestamp but different ids.
	// These will be split across 2 iterators.
	docs := make([]*mongo.OplogDoc, 11)
	for i := 0; i < 10; i++ {
		id := int64(i + 10)
		docs[i] = &mongo.OplogDoc{
			Timestamp:   1,
			OperationId: id,
		}
	}
	// Add one more with a different timestamp.
	docs[10] = &mongo.OplogDoc{
		Timestamp:   2,
		OperationId: 42,
	}
	session := newFakeSession(
		// First block of documents, all time 1
		newFakeIterator(nil, docs[:5]...),
		// Second block, some time 1, one time 2
		newFakeIterator(nil, docs[5:]...),
	)
	tailer := mongo.NewOplogTailer(session, time.Time{})
	defer tailer.Stop()

	for id := int64(10); id < 15; id++ {
		doc := s.getNextOplog(c, tailer)
		c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
		c.Assert(doc.OperationId, gc.Equals, id)
	}

	// Check the query doesn't exclude any in the first request.
	session.checkLastArgs(c, mongo.NewMongoTimestamp(time.Time{}), nil)

	// The OplogTailer will fall off the end of the iterator and get a new one.

	// Ensure that only previously unreported entries are now reported.
	for id := int64(15); id < 20; id++ {
		doc := s.getNextOplog(c, tailer)
		c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
		c.Assert(doc.OperationId, gc.Equals, id)
	}

	// Check we got the next block correctly
	session.checkLastArgs(c, bson.MongoTimestamp(1), []int64{10, 11, 12, 13, 14})

	doc := s.getNextOplog(c, tailer)
	c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(2))
	c.Assert(doc.OperationId, gc.Equals, int64(42))
}

func (s *oplogSuite) TestNoRepeatsAfterIterRestart(c *gc.C) {
	_, session := s.startMongo(c)

	oplog := s.makeFakeOplog(c, session)
	tailer := mongo.NewOplogTailer(oplog, nil, time.Time{})
	defer tailer.Stop()

	// Insert a bunch of oplog entries with the same timestamp (but
	// with different ids) and see them reported.
	for id := int64(10); id < 15; id++ {
		s.insertDoc(c, session, oplog, &mongo.OplogDoc{
			Timestamp:   1,
			OperationId: id,
		})

		doc := s.getNextOplog(c, tailer)
		c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
		c.Assert(doc.OperationId, gc.Equals, id)
	}

	// Force the OplogTailer's iterator to be recreated.
	s.emptyCapped(c, oplog)

	// Reinsert the oplog entries that were already there before and a
	// few more.
	for id := int64(10); id < 20; id++ {
		s.insertDoc(c, session, oplog, &mongo.OplogDoc{
			Timestamp:   1,
			OperationId: id,
		})
	}

	// Insert an entry for a later timestamp.
	s.insertDoc(c, session, oplog, &mongo.OplogDoc{
		Timestamp:   2,
		OperationId: 42,
	})

	// Ensure that only previously unreported entries are now reported.
	for id := int64(15); id < 20; id++ {
		doc := s.getNextOplog(c, tailer)
		c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(1))
		c.Assert(doc.OperationId, gc.Equals, id)
	}

	doc := s.getNextOplog(c, tailer)
	c.Assert(doc.Timestamp, gc.Equals, bson.MongoTimestamp(2))
	c.Assert(doc.OperationId, gc.Equals, int64(42))
}

// ParseTimestampFlag takes in a string the form of <time_t>:<ordinal>,
// where <time_t> is the seconds since the UNIX epoch, and <ordinal> represents
// a counter of operations in the oplog that occurred in the specified second.
// It parses this timestamp string and returns a bson.MongoTimestamp type.
func ParseTimestampFlag(ts string) (bson.MongoTimestamp, error) {
	var seconds, increment int
	timestampFields := strings.Split(ts, ":")
	if len(timestampFields) > 2 {
		return 0, fmt.Errorf("too many : characters")
	}

	seconds, err := strconv.Atoi(timestampFields[0])
	if err != nil {
		return 0, fmt.Errorf("error parsing timestamp seconds: %v", err)
	}

	// parse the increment field if it exists
	if len(timestampFields) == 2 {
		if len(timestampFields[1]) > 0 {
			increment, err = strconv.Atoi(timestampFields[1])
			if err != nil {
				return 0, fmt.Errorf("error parsing timestamp increment: %v", err)
			}
		} else {
			// handle the case where the user writes "<time_t>:" with no ordinal
			increment = 0
		}
	}

	timestamp := (int64(seconds) << 32) | int64(increment)
	return bson.MongoTimestamp(timestamp), nil
}

// WriteLogWithOplog writes out a log record to the a (probably fake)
// oplog collection and the logs collection.
func WriteLogWithOplog(
	oplog *mgo.Collection,
	envUUID string,
	entity names.Tag,
	t time.Time,
	module string,
	location string,
	level loggo.Level,
	msg string,
) error {
	doc := &logDoc{
		Id:       bson.NewObjectId(),
		Time:     t,
		EnvUUID:  envUUID,
		Entity:   entity.String(),
		Module:   module,
		Location: location,
		Level:    level,
		Message:  msg,
	}
	err := oplog.Insert(bson.D{
		{"ts", bson.MongoTimestamp(time.Now().Unix() << 32)}, // an approximation which will do
		{"h", rand.Int63()},                                  // again, a suitable fake
		{"op", "i"},                                          // this will always be an insert
		{"ns", "logs.logs"},
		{"o", doc},
	})
	if err != nil {
		return err
	}

	session := oplog.Database.Session
	logs := session.DB("logs").C("logs")
	return logs.Insert(doc)
}

func (s *oplogSuite) TestNewMongoTimestamp(c *gc.C) {
	t := time.Date(2015, 6, 24, 12, 47, 0, 0, time.FixedZone("somewhere", 5*3600))

	expected := bson.MongoTimestamp(6163845091342417920)
	c.Assert(mongo.NewMongoTimestamp(t), gc.Equals, expected)
	c.Assert(mongo.NewMongoTimestamp(t.In(time.UTC)), gc.Equals, expected)
}

func TailOps(session *mgo.Session, channel OpChan, errChan chan error, options *Options) error {
	s := session.Copy()
	defer s.Close()
	FillEmptyOptions(session, options)
	duration, err := time.ParseDuration(*options.CursorTimeout)
	if err != nil {
		panic("Invalid value for CursorTimeout")
	}
	currTimestamp := options.After(s, options)
	iter := GetOpLogQuery(s, currTimestamp, options).Tail(duration)
	for {
		entry := make(OpLogEntry)
		for iter.Next(entry) {
			op := &Op{"", "", "", nil, bson.MongoTimestamp(0)}
			op.ParseLogEntry(entry)
			if op.Id != "" {
				if options.Filter == nil || options.Filter(op) {
					channel <- op
				}
			}
			currTimestamp = op.Timestamp
		}
		if err = iter.Close(); err != nil {
			errChan <- err
			return err
		}
		if iter.Timeout() {
			continue
		}
		iter = GetOpLogQuery(s, currTimestamp, options).Tail(duration)
	}
	return nil
}

func TestParseLargeBSON(t *testing.T) {
	largeArray := make([]interface{}, 5000)
	for i := 0; i < 5000; i++ {
		largeArray[i] = float64(i)
	}
	expectedOp := map[string]interface{}{
		"ts": bson.MongoTimestamp(6048257058866724865), "h": int64(-6825742652110581687), "v": 2, "op": "i", "ns": "testdb.testdb", "o": map[string]interface{}{
			"_id": bson.ObjectIdHex("53efb9c067fd92348e823860"),
			"val": largeArray}}

	f, err := os.Open("./largetestdata.bson")
	if err != nil {
		t.Fatal("Error loading file", err)
	}
	defer f.Close()
	foundExpectedOp := false
	scanner := New(f)
	for scanner.Scan() {
		op := map[string]interface{}{}
		if err := bson.Unmarshal(scanner.Bytes(), &op); err != nil {
			t.Fatal("Error unmarshalling: ", err)
		}
		if reflect.DeepEqual(op, expectedOp) {
			foundExpectedOp = true
		}
	}
	if scanner.Err() != nil {
		t.Fatal("Scanner error: ", scanner.Err())
	}
	if !foundExpectedOp {
		t.Fatal("Didn't find the expected operation")
	}

}

// NewMongoTimestamp returns a bson.MongoTimestamp repesentation for
// the time.Time given. Note that these timestamps are not the same
// the usual MongoDB time fields. These are an internal format used
// only in a few places such as the replication oplog.
//
// See: http://docs.mongodb.org/manual/reference/bson-types/#timestamps
func NewMongoTimestamp(t time.Time) bson.MongoTimestamp {
	unixTime := t.Unix()
	if unixTime < 0 {
		unixTime = 0
	}
	return bson.MongoTimestamp(unixTime << 32)
}

// get the cursor for the oplog collection, based on the options
// passed in to mongooplog
func buildTailingCursor(oplog *mgo.Collection,
	sourceOptions *SourceOptions) *mgo.Iter {

	// how many seconds in the past we need
	secondsInPast := time.Duration(sourceOptions.Seconds) * time.Second
	// the time threshold for oplog queries
	threshold := time.Now().Add(-secondsInPast)
	// convert to a unix timestamp (seconds since epoch)
	thresholdAsUnix := threshold.Unix()

	// shift it appropriately, to prepare it to be converted to an
	// oplog timestamp
	thresholdShifted := uint64(thresholdAsUnix) << 32

	// build the oplog query
	oplogQuery := bson.M{
		"ts": bson.M{
			"$gte": bson.MongoTimestamp(thresholdShifted),
		},
	}

	// TODO: wait time
	return oplog.Find(oplogQuery).Iter()

}

/* bsonify a mongo Timestamp */
func (m M) timestamp() (timestamp bson.MongoTimestamp, ok bool) {
	if len(m) != 1 {
		return
	}

	if value, contains := m["$timestamp"]; contains {
		if ts, ismap := value.(map[string]interface{}); ismap {
			t, isok := ts["t"]
			if !isok {
				return
			}
			tt, isok := t.(int)
			if !isok {
				return
			}

			i, isok := ts["i"]
			if !isok {
				return
			}
			ii, isok := i.(int)
			if !isok {
				return
			}

			ok = true
			var concat int64
			concat = int64(uint64(tt)<<32 | uint64(ii))
			timestamp = bson.MongoTimestamp(concat)
		}
	}

	return
}

func TestTimestampValue(t *testing.T) {

	Convey("When converting JSON with Timestamp values", t, func() {
		testTS := bson.MongoTimestamp(123456<<32 | 55)

		Convey("works for Timestamp literal", func() {

			jsonMap := map[string]interface{}{
				"ts": json.Timestamp{123456, 55},
			}

			err := ConvertJSONDocumentToBSON(jsonMap)
			So(err, ShouldBeNil)
			So(jsonMap["ts"], ShouldEqual, testTS)
		})

		Convey(`works for Timestamp document`, func() {
			Convey(`{"ts":{"$timestamp":{"t":123456, "i":55}}}`, func() {
				jsonMap := map[string]interface{}{
					"ts": map[string]interface{}{
						"$timestamp": map[string]interface{}{
							"t": 123456.0,
							"i": 55.0,
						},
					},
				}

				bsonMap, err := ConvertJSONValueToBSON(jsonMap)
				So(err, ShouldBeNil)
				So(bsonMap.(map[string]interface{})["ts"], ShouldEqual, testTS)
			})
		})
	})
}