Golang store.NewMessageEntity函数代码示例

func TestFileStoreUpdate(t *testing.T) {
	cleanSnapshot("./snapshot/")
	fs := NewKiteFileStore(".", 5000000, 1*time.Second)
	fs.Start()

	for i := 0; i < 100; i++ {
		//创建消息
		msg := &protocol.BytesMessage{}
		msg.Header = &protocol.Header{
			MessageId:    proto.String(fmt.Sprintf("%x", i) + "26c03f00665862591f696a980b5ac"),
			Topic:        proto.String("trade"),
			MessageType:  proto.String("pay-succ"),
			ExpiredTime:  proto.Int64(time.Now().Add(10 * time.Minute).Unix()),
			DeliverLimit: proto.Int32(100),
			GroupId:      proto.String("go-kite-test"),
			Commit:       proto.Bool(true),
			Fly:          proto.Bool(false)}
		msg.Body = []byte("hello world")

		entity := store.NewMessageEntity(protocol.NewQMessage(msg))
		// log.Printf("------------%s", entity.Header)
		succ := fs.Save(entity)
		if !succ {
			t.Fail()
		}
	}

	//commit and check
	for i := 0; i < 100; i++ {
		id := fmt.Sprintf("%x", i) + "26c03f00665862591f696a980b5ac"

		//创建消息
		msg := &store.MessageEntity{
			MessageId:    id,
			DeliverCount: 1,
			SuccGroups:   []string{},
			FailGroups:   []string{"s-mts-test"}}

		succ := fs.UpdateEntity(msg)
		if !succ {
			t.Fail()
		}
		//check entity
		entity := fs.Query(id)
		// log.Printf("++++++++++++++|%s|%s", entity.Header, string(entity.GetBody().([]byte)))
		if nil == entity {
			t.Fail()
		} else if !entity.Commit && entity.DeliverCount != 1 &&
			entity.FailGroups[0] != "s-mts-test" {
			t.Fail()
			// log.Println(entity)
		}
	}
	fs.Stop()
	// cleanSnapshot("./snapshot/")
}

func TestConvert2Entity(t *testing.T) {

	options := MysqlOptions{
		Addr:         "localhost:3306",
		Username:     "root",
		Password:     "",
		ShardNum:     4,
		BatchUpSize:  1000,
		BatchDelSize: 1000,
		FlushPeriod:  1 * time.Minute,
		MaxIdleConn:  2,
		MaxOpenConn:  4}

	hs := newDbShard(options)

	c := convertor{}
	sqlwrapper := newSqlwrapper("kite_msg", hs, store.MessageEntity{})
	sqlwrapper.initSQL()
	c.columns = sqlwrapper.columns

	//创建消息
	msg := &protocol.BytesMessage{}
	msg.Header = &protocol.Header{
		MessageId:    proto.String("26c03f00665462591f696a980b5a6c4"),
		Topic:        proto.String("trade"),
		MessageType:  proto.String("pay-succ"),
		ExpiredTime:  proto.Int64(time.Now().Add(10 * time.Minute).Unix()),
		DeliverLimit: proto.Int32(100),
		GroupId:      proto.String("go-kite-test"),
		Commit:       proto.Bool(false),
		Fly:          proto.Bool(false)}
	msg.Body = []byte("hello world")

	entity := store.NewMessageEntity(protocol.NewQMessage(msg))
	entity.SuccGroups = []string{"go-kite-test"}
	hn, _ := os.Hostname()
	entity.KiteServer = hn

	params := c.Convert2Params(entity)
	t.Logf("TestConvert2Entity|Convert2Params|%s\n", params)
	econ := &store.MessageEntity{}
	c.Convert2Entity(params, econ, func(colname string) bool {
		return false
	})
	t.Logf("TestConvert2Entity|Convert2Entity|%s\n", econ)
	if econ.MessageId != entity.MessageId {
		t.Fail()
	}

	if econ.ExpiredTime != entity.ExpiredTime {
		t.Fail()
	}

}

func (self *AcceptHandler) Process(ctx *pipe.DefaultPipelineContext, event pipe.IEvent) error {
	// log.Debug("AcceptHandler|Process|%s|%t\n", self.GetName(), event)

	ae, ok := self.cast(event)
	if !ok {
		return pipe.ERROR_INVALID_EVENT_TYPE
	}
	//这里处理一下ae,做一下校验
	var msg *store.MessageEntity
	switch ae.msgType {
	case protocol.CMD_DELIVER_ACK:
		//收到投递结果直接attach响应
		// log.DebugLog("kite_handler", "AcceptHandler|DELIVER_ACK|%s|%t", ae.opaque, ae.msg)
		ae.remoteClient.Attach(ae.opaque, ae.msg)
		return nil
	case protocol.CMD_HEARTBEAT:
		hb := ae.msg.(*protocol.HeartBeat)
		event = pipe.NewHeartbeatEvent(ae.remoteClient, ae.opaque, hb.GetVersion())
		ctx.SendForward(event)
		return nil

	case protocol.CMD_BYTES_MESSAGE:
		msg = store.NewMessageEntity(protocol.NewQMessage(ae.msg.(*protocol.BytesMessage)))
	case protocol.CMD_STRING_MESSAGE:
		msg = store.NewMessageEntity(protocol.NewQMessage(ae.msg.(*protocol.StringMessage)))
	default:
		//这只是一个bug不支持的数据类型能给你
		log.WarnLog("kite_handler", "AcceptHandler|Process|%s|%t", INVALID_MSG_TYPE_ERROR, ae.msg)
	}

	if nil != msg {
		msg.PublishTime = time.Now().Unix()
		msg.KiteServer = self.kiteserver
		deliver := newPersistentEvent(msg, ae.remoteClient, ae.opaque)
		ctx.SendForward(deliver)
		return nil
	}
	return INVALID_MSG_TYPE_ERROR
}

func TestFileStoreDelete(t *testing.T) {

	cleanSnapshot("./snapshot/")
	fs := NewKiteFileStore(".", 5000000, 1*time.Second)
	fs.Start()

	for i := 0; i < 100; i++ {
		//创建消息
		msg := &protocol.BytesMessage{}
		msg.Header = &protocol.Header{
			MessageId:    proto.String(fmt.Sprintf("%x", i) + "26c03f00665862591f696a980b5ac"),
			Topic:        proto.String("trade"),
			MessageType:  proto.String("pay-succ"),
			ExpiredTime:  proto.Int64(time.Now().Add(10 * time.Minute).Unix()),
			DeliverLimit: proto.Int32(100),
			GroupId:      proto.String("go-kite-test"),
			Commit:       proto.Bool(true),
			Fly:          proto.Bool(false)}
		msg.Body = []byte("hello world")

		entity := store.NewMessageEntity(protocol.NewQMessage(msg))

		succ := fs.Save(entity)
		if !succ {
			t.Fail()
		}
	}

	//commit and check
	for i := 0; i < 100; i++ {
		id := fmt.Sprintf("%x", i) + "26c03f00665862591f696a980b5ac"

		//delete
		fs.Delete(id)

		//check entity
		entity := fs.Query(id)
		if nil != entity {
			t.Fail()
		}
	}
	fs.Stop()
	cleanSnapshot("./snapshot/")
}

func TestRecoverManager(t *testing.T) {

	pipeline := NewDefaultPipeline()

	kitedb := memory.NewKiteMemoryStore(100, 100)

	messageid := store.MessageId()
	t.Logf("messageid:%s\b", messageid)
	entity := store.NewMessageEntity(protocol.NewQMessage(buildStringMessage(messageid)))
	kitedb.Save(entity)
	go func() {
		for {
			log.Println(kitedb.Monitor())
			time.Sleep(1 * time.Second)
		}
	}()

	fs := stat.NewFlowStat("recover")
	fs.Start()
	ch := make(chan bool, 1)

	// 临时在这里创建的BindExchanger
	exchanger := binding.NewBindExchanger("localhost:2181", "127.0.0.1:13800")
	pipeline.RegisteHandler("deliverpre", handler.NewDeliverPreHandler("deliverpre", kitedb, exchanger, fs, 100))
	pipeline.RegisteHandler("deliver", newmockDeliverHandler("deliver", ch))
	hostname, _ := os.Hostname()
	rm := NewRecoverManager(hostname, 16*time.Second, pipeline, kitedb)
	rm.Start()
	select {
	case succ := <-ch:
		log.Printf("--------------recover %s\n", succ)
	case <-time.After(5 * time.Second):
		t.Fail()
		log.Println("waite recover  deliver timeout\n")
	}

}

func innerT(kiteMysql *KiteMysqlStore, msg interface{}, msgid string, t *testing.T) {

	qm := protocol.NewQMessage(msg)
	entity := store.NewMessageEntity(qm)
	entity.SuccGroups = []string{"go-kite-test"}
	hn := "localhost"
	entity.KiteServer = hn
	entity.PublishTime = time.Now().Unix()

	succ := kiteMysql.Save(entity)
	if !succ {
		t.Fail()
	} else {
		t.Logf("SAVE|SUCC|%s\n", entity)
	}

	ret := kiteMysql.Query(msgid)
	t.Logf("Query|%s|%s\n", msgid, ret)
	if nil == ret {
		t.Fail()
		return
	}

	bb, ok := qm.GetBody().([]byte)
	if ok {
		rb, _ := ret.GetBody().([]byte)
		if string(rb) != string(bb) {
			t.Fail()
		} else {
			t.Logf("Query|SUCC|%s\n", ret)
		}
	} else {
		bs, _ := qm.GetBody().(string)
		rs, _ := ret.GetBody().(string)
		if bs != rs {
			t.Fail()
		} else {
			t.Logf("Query|SUCC|%s\n", ret)
		}
	}

	t.Logf("Commint BEGIN")
	commit := kiteMysql.Commit(msgid)
	if !commit {
		t.Logf("Commint FAIL")
		t.Fail()
	}
	t.Logf("Commint END")
	time.Sleep(200 * time.Millisecond)
	ret = kiteMysql.Query(msgid)
	t.Logf("PageQueryEntity|COMMIT RESULT|%s\n", ret)
	if !ret.Commit {
		t.Logf("Commit|FAIL|%s\n", ret)
		t.Fail()
	}

	hasNext, entities := kiteMysql.PageQueryEntity(msgid, hn, time.Now().Unix(), 0, 10)
	t.Logf("PageQueryEntity|%s\n", entities)
	if hasNext {
		t.Logf("PageQueryEntity|FAIL|HasNext|%s\n", entities)
		t.Fail()
	} else {
		if len(entities) != 1 {
			t.Logf("PageQueryEntity|FAIL|%s\n", entities)
			t.Fail()
		} else {
			if entities[0].Header.GetMessageId() != qm.GetHeader().GetMessageId() {
				t.Fail()
			}
		}
	}
}

//增加迁移
func TestExpiredDLQ(t *testing.T) {

	options := MysqlOptions{
		Addr:         "localhost:3306",
		DB:           "kite",
		Username:     "root",
		Password:     "",
		ShardNum:     4,
		BatchUpSize:  100,
		BatchDelSize: 100,
		FlushPeriod:  10 * time.Millisecond,
		MaxIdleConn:  10,
		MaxOpenConn:  10}

	kiteMysql := NewKiteMysql(options, "localhost")
	truncate(kiteMysql)
	messageIds := make([]string, 0, 10)
	for i := 0; i < 256; i++ {
		//创建消息
		msg := &protocol.BytesMessage{}
		msg.Header = &protocol.Header{
			MessageId:    proto.String("26c03f00665862591f696a980b5a6" + fmt.Sprintf("%x", i)),
			Topic:        proto.String("trade"),
			MessageType:  proto.String("pay-succ"),
			ExpiredTime:  proto.Int64(time.Now().Add(10 * time.Second).Unix()),
			DeliverLimit: proto.Int32(100),
			GroupId:      proto.String("go-kite-test"),
			Commit:       proto.Bool(false),
			Fly:          proto.Bool(false)}

		msg.Body = []byte("hello world")
		qm := protocol.NewQMessage(msg)
		entity := store.NewMessageEntity(qm)
		entity.SuccGroups = []string{"go-kite-test"}
		hn := "localhost"
		entity.KiteServer = hn
		entity.PublishTime = time.Now().Unix()
		entity.DeliverCount = 101

		succ := kiteMysql.Save(entity)
		if !succ {
			t.Fail()
		} else {
			// fmt.Printf("SAVE|SUCC|%s\n", entity)
		}
		messageIds = append(messageIds, msg.GetHeader().GetMessageId())
	}

	//开始清理
	kiteMysql.MoveExpired()

	for _, messageId := range messageIds {

		entity := kiteMysql.Query(messageId)
		if nil != entity {
			t.Fail()
			fmt.Println("MoveExpired|FAIL|" + messageId)
		}
	}

	total := 0
	for i := 0; i < 4; i++ {
		db := kiteMysql.dbshard.FindSlave(fmt.Sprintf("%x", i))
		rows, _ := db.Query("select count(*) from kite_msg_dlq")
		if rows.Next() {
			var count int
			rows.Scan(&count)
			t.Logf("TestExpiredDLQ|COUNT|%s|%d\n", fmt.Sprintf("%x", i), total)
			total += count
		}
	}

	if total != 256 {
		t.Fail()
		t.Logf("TestExpiredDLQ|TOTAL NOT EQUAL|%d\n", total)
	}

	truncate(kiteMysql)
	kiteMysql.Stop()
}

func TestPageQuery(t *testing.T) {

	options := MysqlOptions{
		Addr:         "localhost:3306",
		DB:           "kite",
		Username:     "root",
		Password:     "",
		ShardNum:     4,
		BatchUpSize:  100,
		BatchDelSize: 100,
		FlushPeriod:  10 * time.Millisecond,
		MaxIdleConn:  10,
		MaxOpenConn:  10}

	kiteMysql := NewKiteMysql(options, "localhost")
	truncate(kiteMysql)
	hn := "localhost"
	for i := 0; i < 10; i++ {
		//创建消息
		msg := &protocol.BytesMessage{}
		msg.Header = &protocol.Header{
			MessageId:    proto.String(fmt.Sprintf("%x", i) + "26c03f00665862591f696a980b5ac"),
			Topic:        proto.String("trade"),
			MessageType:  proto.String("pay-succ"),
			ExpiredTime:  proto.Int64(time.Now().Add(10 * time.Minute).Unix()),
			DeliverLimit: proto.Int32(100),
			GroupId:      proto.String("go-kite-test"),
			Commit:       proto.Bool(false),
			Fly:          proto.Bool(false)}
		msg.Body = []byte("hello world")

		entity := store.NewMessageEntity(protocol.NewQMessage(msg))

		entity.KiteServer = hn
		entity.PublishTime = time.Now().Unix()
		kiteMysql.Save(entity)
	}

	startIdx := 0
	hasMore := true
	count := 0
	//开始分页查询未过期的消息实体
	for hasMore {
		more, entities := kiteMysql.PageQueryEntity("c", hn,
			time.Now().Unix(), 0, 1)
		if len(entities) <= 0 {
			break
		}

		//开始发起重投
		for _, entity := range entities {
			count++
			t.Logf("TestPageQuery|PageQueryEntity|%s\n", entity.MessageId)
			msg := &store.MessageEntity{
				MessageId:    entity.MessageId,
				DeliverCount: 1,
				SuccGroups:   []string{},
				FailGroups:   []string{"s-mts-test"},
				//设置一下下一次投递时间
				NextDeliverTime: time.Now().Add(1 * time.Minute).Unix()}
			kiteMysql.AsyncUpdate(msg)

		}

		time.Sleep(1 * time.Second)
		hasMore = more
		startIdx += len(entities)
	}
	if count != 10 {
		t.Fail()
		t.Logf("TestPageQuery|IDX|FAIL|%d\n", count)
		return
	}

	startIdx = 0
	hasMore = true
	//开始分页查询未过期的消息实体
	for hasMore {
		more, entities := kiteMysql.PageQueryEntity("6c", hn,
			time.Now().Add(8*time.Minute).Unix(), startIdx, 1)
		if len(entities) <= 0 {
			t.Logf("TestPageQuery|CHECK|NO DATA|%s\n", entities)
			break
		}

		//开始发起重投
		for _, entity := range entities {
			if entity.DeliverCount != 1 || entity.FailGroups[0] != "s-mts-test" {
				t.Fail()
			}
			t.Logf("TestPageQuery|PageQueryEntity|CHECK|%s\n", entity.MessageId)
		}
		startIdx += len(entities)
		hasMore = more
	}

	t.Logf("TestPageQuery|CHECK|%d\n", startIdx)
	if startIdx != 10 {
		t.Fail()
	}
//.........这里部分代码省略.........

func TestBatch(t *testing.T) {
	options := MysqlOptions{
		Addr:         "localhost:3306",
		DB:           "kite",
		Username:     "root",
		Password:     "",
		ShardNum:     4,
		BatchUpSize:  100,
		BatchDelSize: 100,
		FlushPeriod:  10 * time.Millisecond,
		MaxIdleConn:  10,
		MaxOpenConn:  10}

	kiteMysql := NewKiteMysql(options, "localhost")

	truncate(kiteMysql)

	mids := make([]string, 0, 16)
	for i := 0; i < 16; i++ {
		//创建消息
		msg := &protocol.BytesMessage{}
		msg.Header = &protocol.Header{
			MessageId:    proto.String("26c03f00665862591f696a980b5a6" + fmt.Sprintf("%x", i)),
			Topic:        proto.String("trade"),
			MessageType:  proto.String("pay-succ"),
			ExpiredTime:  proto.Int64(time.Now().Add(10 * time.Minute).Unix()),
			DeliverLimit: proto.Int32(100),
			GroupId:      proto.String("go-kite-test"),
			Commit:       proto.Bool(false),
			Fly:          proto.Bool(false)}
		msg.Body = []byte("hello world")

		entity := store.NewMessageEntity(protocol.NewQMessage(msg))
		entity.SuccGroups = []string{"go-kite-test"}
		hn := "localhost"
		entity.KiteServer = hn
		entity.PublishTime = time.Now().Unix()
		kiteMysql.Save(entity)
		mids = append(mids, entity.MessageId)
	}

	for _, v := range mids {

		msg := &store.MessageEntity{
			MessageId:    v,
			DeliverCount: 1,
			SuccGroups:   []string{"s-mts-test"},
			FailGroups:   []string{},
			//设置一下下一次投递时间
			NextDeliverTime: time.Now().Unix()}
		kiteMysql.AsyncUpdate(msg)
	}

	time.Sleep(5 * time.Second)
	for _, v := range mids {
		e := kiteMysql.Query(v)
		if nil == e || len(e.SuccGroups) < 1 {
			t.Fatalf("TestBatch|Update FAIL|%s|%s\n", e, v)
			t.Fail()
			return
		}
		t.Logf("Query|%s\n", e)
	}

	//测试批量删除
	for _, v := range mids {
		kiteMysql.AsyncDelete(v)
	}
	time.Sleep(5 * time.Second)
	for _, v := range mids {
		entity := kiteMysql.Query(v)
		if nil != entity {
			t.Fatalf("TestBatch|AysncDelete FAIL|%s\n", entity)
			t.Fail()

		}
	}

	truncate(kiteMysql)
}

func TestFileStoreInit(t *testing.T) {

	cleanSnapshot("./snapshot/")
	fs := NewKiteFileStore(".", 5000000, 1*time.Second)
	fs.Start()

	for i := 0; i < 100; i++ {
		//创建消息
		msg := &protocol.BytesMessage{}
		msg.Header = &protocol.Header{
			MessageId:    proto.String(fmt.Sprint(i) + "26c03f00665862591f696a980b5ac"),
			Topic:        proto.String("trade"),
			MessageType:  proto.String("pay-succ"),
			ExpiredTime:  proto.Int64(time.Now().Add(10 * time.Minute).Unix()),
			DeliverLimit: proto.Int32(100),
			GroupId:      proto.String("go-kite-test"),
			Commit:       proto.Bool(true),
			Fly:          proto.Bool(false)}
		msg.Body = []byte("hello world")

		entity := store.NewMessageEntity(protocol.NewQMessage(msg))

		succ := fs.Save(entity)
		if !succ {
			t.Fail()
		}

		if i < 50 {
			fs.AsyncDelete(entity.MessageId)
		}
	}

	fs.Stop()

	time.Sleep(10 * time.Second)
	log.Println("-------------------Query")
	fs = NewKiteFileStore(".", 5000000, 1*time.Second)
	fs.Start()

	for _, v := range fs.oplogs {
		for _, ol := range v {
			ob := ol.Value.(*opBody)
			log.Printf("TestFileStoreInit|Check|%d|%s", ob.Id, ob.MessageId)
		}
	}
	log.Printf("TestFileStoreInit|Check|SUCC|\n")
	//commit and check
	for i := 50; i < 100; i++ {
		id := fmt.Sprint(i) + "26c03f00665862591f696a980b5ac"

		//check entity
		entity := fs.Query(id)
		if nil == entity || !entity.Commit {
			log.Printf("TestFileStoreInit|Exist|FAIL|%s|%s", id, entity)
			t.Fail()
			return
		}
		log.Printf("TestFileStoreInit|Exist|SUCC|%d|%s|%s", i, id, entity)
	}

	log.Printf("TestFileStoreInit|Exist|\n")

	//commit and check
	for i := 0; i < 50; i++ {
		id := fmt.Sprint(i) + "26c03f00665862591f696a980b5ac"

		//check entity
		entity := fs.Query(id)
		if nil != entity {
			log.Printf("TestFileStoreInit|Delete|FAIL|%s", id)
			t.Fail()
		}
	}

	log.Printf("TestFileStoreInit|Delete\n")
	fs.Stop()
	cleanSnapshot("./snapshot/")
}

func TestConvert2Params(t *testing.T) {

	options := MysqlOptions{
		Addr:         "localhost:3306",
		Username:     "root",
		Password:     "",
		ShardNum:     4,
		BatchUpSize:  1000,
		BatchDelSize: 1000,
		FlushPeriod:  1 * time.Minute,
		MaxIdleConn:  2,
		MaxOpenConn:  4}

	hs := newDbShard(options)

	c := convertor{}
	sqlwrapper := newSqlwrapper("kite_msg", hs, store.MessageEntity{})
	sqlwrapper.initSQL()
	c.columns = sqlwrapper.columns

	//创建消息
	msg := &protocol.BytesMessage{}
	msg.Header = &protocol.Header{
		MessageId:    proto.String("26c03f00665862591f696a980b5a6c4"),
		Topic:        proto.String("trade"),
		MessageType:  proto.String("pay-succ"),
		ExpiredTime:  proto.Int64(time.Now().Add(10 * time.Minute).Unix()),
		DeliverLimit: proto.Int32(100),
		GroupId:      proto.String("go-kite-test"),
		Commit:       proto.Bool(false),
		Fly:          proto.Bool(false)}
	msg.Body = []byte("hello world")

	entity := store.NewMessageEntity(protocol.NewQMessage(msg))
	entity.SuccGroups = []string{"go-kite-test"}
	hn, _ := os.Hostname()
	entity.KiteServer = hn

	params := c.Convert2Params(entity)

	if nil != params {
		for i, col := range c.columns {
			cv := params[i]
			t.Logf("TestConvert2Params|FIELD|%s|%s\n", col.fieldName, cv)
			if col.fieldName == "MessageId" {
				rv := reflect.ValueOf(cv)
				s := rv.Elem().Interface()
				if s.(string) != entity.MessageId {
					t.Fail()
				}
			} else if col.columnName == "body" {
				rv := reflect.ValueOf(cv)
				s := rv.Elem().Interface()
				if string(s.([]byte)) != string(entity.GetBody().([]byte)) {
					t.Fail()
				}
			}
		}
	} else {
		t.Fail()
	}
}