Golang stdlog.Printf函数代码示例

// redis-cli对monitor指令进行特殊处理，只要monitor不断输出StatusReply，可以实现不间断的流输出
// 适用于海量数据的扫描输出，比如iterator扫描整个数据库
func (server *GoRedisServer) OnMONITOR(session *Session, cmd *Command) (reply *Reply) {
	// 特殊使用，monitor输出全部key
	if cmd.Len() > 1 {
		switch strings.ToUpper(cmd.StringAtIndex(1)) {
		case "KEYS":
			server.monitorKeys(session, cmd)
		default:
			reply = ErrorReply("bad monitor command")
			go func() {
				time.Sleep(time.Millisecond * 100)
				session.Close()
			}()
		}
		return
	}

	session.WriteReply(StatusReply("OK"))
	client := NewMonClient(session)
	remoteHost := session.RemoteAddr().String()

	go func() {
		stdlog.Printf("[%s] monitor start\n", remoteHost)
		// sleep一下，避免启动瞬间输出 +1394530022.495448 [0 127.0.0.1:51980] "monitor"
		time.Sleep(time.Millisecond * 10)
		server.monmgr.Put(remoteHost, client)
		client.Start()
		server.monmgr.Remove(remoteHost)
		stdlog.Printf("[%s] monitor exit\n", remoteHost)
	}()

	return
}

// ./slaveof-proxy -src localhost:6379 -dest remote:6379 -pullrate 400 -pushrate 400
func main() {
	runtime.GOMAXPROCS(4)
	src := flag.String("src", "", "master")
	dest := flag.String("dest", "", "slave")
	pullrate := flag.Int("pullrate", 400, "pull rate in Mbits/s")
	pushrate := flag.Int("pushrate", 400, "push rate in Mbits/s")
	buffer := flag.Int("buffer", 100, "buffer x10000 records")
	flag.Parse()

	if *pullrate < 100 {
		*pullrate = 100
	}
	if *pushrate < 100 {
		*pushrate = 100
	}
	if *buffer < 100 {
		*buffer = 100
	} else if *buffer > 1000 {
		*buffer = 1000
	}

	stdlog.Println("slaveof-proxy 1.0.3")
	stdlog.Printf("from [%s] to [%s]\n", *src, *dest)
	stdlog.Printf("pull [%d] buffer [%d]\n", *pullrate, *buffer)
	stdlog.Println("SYNC ...")

	client, err := NewClient(*src, *dest, *buffer)
	if err != nil {
		stdlog.Println("ERR", err)
		return
	}
	client.SetPullRate(*pullrate / 8 * 1024 * 1024)
	client.SetPushRate(*pushrate / 8 * 1024 * 1024)
	client.Sync()
}

// nextseq，返回快照的下一条seq位置
func (server *GoRedisServer) sendSnapshot(session *Session) (nextseq int64, err error) {
	server.Suspend()                     //挂起全部操作
	snap := server.levelRedis.Snapshot() // 挂起后建立只读快照
	defer snap.Close()                   // 必须释放
	lastseq := server.synclog.MaxSeq()   // 获取当前日志序号
	server.Resume()                      // 唤醒，如果不调用Resume，整个服务器无法继续工作

	if err = session.WriteCommand(NewCommand([]byte("SYNC_RAW_START"))); err != nil {
		return
	}

	// scan snapshot
	broken := false
	sendcount := 0
	sendfinish := false
	go func() {
		for {
			time.Sleep(time.Second * 1)
			if sendcount == -1 {
				break // finish
			} else if broken {
				break // cancel
			}
			if sendfinish {
				stdlog.Printf("[S %s] snap send finish, %d raw items\n", session.GetAttribute(S_HOST), sendcount)
				break
			} else {
				stdlog.Printf("[S %s] snap send %d raw items\n", session.GetAttribute(S_HOST), sendcount)
			}
		}
	}()

	// gogogo
	snap.RangeEnumerate([]byte{}, []byte{levelredis.MAXBYTE}, levelredis.IterForward, func(i int, key, value []byte, quit *bool) {
		if bytes.HasPrefix(key, []byte(PREFIX)) {
			return
		}
		cmd := NewCommand([]byte("SYNC_RAW"), key, value)
		err = session.WriteCommand(cmd)
		if err != nil {
			broken = true
			*quit = true
		}
		sendcount++
	})

	if broken {
		return -1, err
	}

	sendfinish = true

	curseq := server.synclog.MaxSeq()
	if err = session.WriteCommand(NewCommand(formatByteSlice("SYNC_RAW_END", sendcount, lastseq, curseq)...)); err != nil {
		return
	}
	nextseq = lastseq + 1
	return nextseq, nil
}

// 发起主从同步请求
func (server *GoRedisServer) initSlaveOf() {
	host, port := server.opt.SlaveOf()
	if len(host) > 0 && port != 0 {
		stdlog.Printf("init slaveof %s:%d\n", host, port)
		// 模拟外部, session=nil
		simulatedCmd := NewCommand(formatByteSlice("SLAVEOF", host, port)...)
		reply := server.OnSLAVEOF(nil, simulatedCmd)
		stdlog.Printf("slaveof: %s:%d, %s\n", host, port, reply)
	}
}

func (s *SlaveClient) RdbRecvFinishCallback(r *bufio.Reader) {
	stdlog.Printf("[M %s] rdb recv finish, start decoding... \n", s.src.RemoteAddr())
	s.initlog()
	go s.readAllReply()
	// decode
	dec := newRdbDecoder(s)
	err := rdb.Decode(r, dec)
	if err != nil {
		// must cancel
		stdlog.Printf("[M %s] decode error %s\n", s.src.RemoteAddr(), err)
		s.Close()
	}
	return
}

// S: SYNC UID [UID] PORT [PORT] SNAP [1/0] SEQ [-1/...]
func (server *GoRedisServer) OnSYNC(session *Session, cmd *Command) (reply *Reply) {
	stdlog.Printf("[S %s] %s\n", session.RemoteAddr(), cmd)

	args := cmd.Args()[1:]
	if len(args) < 2 || len(args)%2 != 0 {
		session.Close()
		return
	}
	for i := 0; i < len(args); i += 2 {
		session.SetAttribute(string(args[i]), string(args[i+1]))
	}

	if !server.synclog.IsEnabled() {
		stdlog.Println("synclog enable")
		server.synclog.Enable()
	}

	// 使用从库端口代替socket端口，标识来源
	h, _ := splitHostPort(session.RemoteAddr().String())
	remoteHost := fmt.Sprintf("%s:%s", h, session.GetAttribute("PORT"))
	session.SetAttribute(S_HOST, remoteHost)
	session.SetAttribute(S_STATUS, REPL_WAIT)

	go func() {
		server.syncmgr.Put(remoteHost, session)
		err := server.doSync(session, cmd)
		if err != nil {
			stdlog.Println("sync ", err)
		}
		session.Close()
		server.syncmgr.Remove(remoteHost)
	}()

	return NOREPLY
}

// ServerHandler.On()
// 由GoRedis协议层触发，通过反射调用OnGET/OnSET等方法
func (server *GoRedisServer) On(session *Session, cmd *Command) (reply *Reply) {
	// invoke & time
	begin := time.Now()

	// suspend & resume
	server.rwlock.Lock()
	server.rwlock.Unlock()

	cmd.SetAttribute(C_SESSION, session)

	// varify command
	if err := verifyCommand(cmd); err != nil {
		stdlog.Printf("[%s] bad command %s\n", session.RemoteAddr(), cmd)
		return ErrorReply(err)
	}

	// invoke
	reply = server.invokeCommandHandler(session, cmd)

	elapsed := time.Now().Sub(begin)
	cmd.SetAttribute(C_ELAPSED, elapsed)

	// async: counter/sync/monitor
	server.rwwait.Add(1)
	server.cmdChan <- cmd

	return
}

func (server *GoRedisServer) OnAOF(session *Session, cmd *Command) (reply *Reply) {
	defer func() {
		if v := recover(); v != nil {
			stdlog.Printf("aof panic %s\n", cmd)
			stdlog.Println(string(debug.Stack()))
		}
	}()

	onoff := strings.ToUpper(cmd.StringAtIndex(1))
	if onoff == "YES" {
		if server.aofwriter != nil {
			return ErrorReply("aof already inited")
		}
		if !server.synclog.IsEnabled() {
			stdlog.Println("synclog enable")
			server.synclog.Enable()
		}
		go func() {
			err := server.aofStart()
			if err != nil {
				stdlog.Println("aof", err)
			}
		}()
	} else if onoff == "NO" {
		return server.onAOF_NO()
	} else {
		return ErrorReply("must be YES/NO")
	}
	return StatusReply("OK")
}

func (server *GoRedisServer) Init() (err error) {

	server.initSignalNotify()

	stdlog.Println("server init, version", VERSION, "...")
	err = server.initLevelDB()
	if err != nil {
		return
	}
	err = server.initSyncLog()
	if err != nil {
		return
	}
	server.config = NewConfig(server.levelRedis, PREFIX+"config:")
	// monitor
	server.initCommandMonitor(server.directory + "/cmd.log")
	server.initCommandCounterLog("string", []string{"GET", "SET", "MGET", "MSET", "INCR", "DECR", "INCRBY", "DECRBY"})
	server.initCommandCounterLog("hash", []string{"HGETALL", "HGET", "HSET", "HDEL", "HMGET", "HMSET", "HINCRBY", "HLEN"})
	server.initCommandCounterLog("set", []string{"SADD", "SCARD", "SISMEMBER", "SMEMBERS", "SREM"})
	server.initCommandCounterLog("list", []string{"LPUSH", "RPUSH", "LPOP", "RPOP", "LINDEX", "LLEN", "LRANGE", "LTRIM"})
	server.initCommandCounterLog("zset", []string{"ZADD", "ZCARD", "ZSCORE", "ZINCRBY", "ZRANGE", "ZRANGEBYSCORE", "ZRANK", "ZREM", "ZREMRANGEBYRANK", "ZREMRANGEBYSCORE", "ZREVRANGE", "ZREVRANGEBYSCORE", "ZREVRANK"})
	server.initSeqLog(server.directory + "/seq.log")
	server.initLeveldbIOLog(server.directory + "/leveldb.io.log")
	server.initLeveldbStatsLog(server.directory + "/leveldb.stats.log")
	server.initExecLog(server.directory + "/exec.time.log")
	server.initSlowlog(server.directory + "/slow.log")
	stdlog.Printf("init uid %s\n", server.UID())
	server.initSlaveOf()
	return
}

// 从主库获取数据
func (server *GoRedisServer) OnSLAVEOF(session *Session, cmd *Command) (reply *Reply) {
	// 保障不会奔溃
	defer func() {
		if v := recover(); v != nil {
			stdlog.Printf("[%s] slaveof panic %s\n", session.RemoteAddr(), cmd)
			stdlog.Println(string(debug.Stack()))
		}
	}()
	arg1, arg2 := cmd.StringAtIndex(1), cmd.StringAtIndex(2)
	// SLAVEOF NO ONE
	if strings.ToUpper(arg1) == "NO" && strings.ToUpper(arg2) == "ONE" {
		return server.onSlaveOfNoOne(session, cmd)
	}

	// connect to master
	hostPort := arg1 + ":" + arg2
	conn, err := net.Dial("tcp", hostPort)
	if err != nil {
		return ErrorReply(err)
	}

	// check exists
	remoteHost := conn.RemoteAddr().String()
	if server.slavemgr.Contains(remoteHost) {
		return ErrorReply("connection exists")
	}

	masterSession := NewSession(conn)
	isgoredis, version, err := redisInfo(masterSession)
	if err != nil {
		return ErrorReply(err)
	}

	var client ISlaveClient
	if isgoredis {
		slavelog.Printf("[M %s] SLAVEOF %s GoRedis:%s\n", remoteHost, remoteHost, version)
		if client, err = NewSlaveClientV2(server, masterSession); err != nil {
			return ErrorReply(err)
		}
	} else {
		slavelog.Printf("[M %s] SLAVEOF %s Redis:%s\n", remoteHost, remoteHost, version)
		if client, err = NewSlaveClient(server, masterSession); err != nil {
			return ErrorReply(err)
		}
	}

	// async
	go func() {
		client.Session().SetAttribute(S_STATUS, REPL_WAIT)
		server.slavemgr.Put(remoteHost, client)
		err := client.Sync()
		if err != nil {
			slavelog.Printf("[M %s] sync broken %s\n", remoteHost, err)
		}
		client.Close()
		server.slavemgr.Remove(remoteHost)
	}()

	return StatusReply("OK")
}

// ./slaveof-proxy -src localhost:6379 -dest remote:6379 -pullrate 400 -pushrate 400
func main() {
	runtime.GOMAXPROCS(4)
	src := flag.String("src", "", "master")
	dest := flag.String("dest", "", "slave")
	pullrate := flag.Int("pullrate", 400, "pull rate in Mbits/s")
	pushrate := flag.Int("pushrate", 400, "push rate in Mbits/s")
	buffer := flag.Int("buffer", 100, "buffer x10000 records")
	dbpath := flag.String("dbpath", "/tmp", "rdb path")
	flag.Parse()

	if *pullrate < 100 {
		*pullrate = 100
	}
	if *pushrate < 100 {
		*pushrate = 100
	}
	if *buffer < 100 {
		*buffer = 100
	} else if *buffer > 1000 {
		*buffer = 1000
	}

	stdlog.Println("slaveof-proxy 1.0.4")
	if len(*src) == 0 || len(*dest) == 0 {
		stdlog.Println("Usage: ./slaveof-proxy -src master:port -dest slave:6379 -pullrate 400 -pushrate 400 -buffer 100 -dbpath /tmp")
		return
	}

	stdlog.Printf("from [%s] to [%s]\n", *src, *dest)
	stdlog.Printf("pull [%d] buffer [%d]\n", *pullrate, *buffer)
	stdlog.Println("SYNC ...")

	client, err := NewClient(*src, *dest, *buffer)
	if err != nil {
		stdlog.Println("ERR", err)
		return
	}
	client.SetPullRate(*pullrate / 8 * 1024 * 1024)
	client.SetPushRate(*pushrate / 8 * 1024 * 1024)
	client.SetDbPath(*dbpath)
	err = client.Sync()
	if err != nil {
		panic(err)
	}
}

func (server *GoRedisServer) doSync(session *Session, cmd *Command) (err error) {
	// snapshot
	var nextseq int64
	if session.GetAttribute("SEQ") != nil {
		nextseq, err = strconv.ParseInt(session.GetAttribute("SEQ").(string), 10, 64)
		if err != nil {
			return
		}
	}

	remoteHost := session.GetAttribute(S_HOST).(string)

	if session.GetAttribute("SNAP") != nil && session.GetAttribute("SNAP").(string) == "1" {
		session.SetAttribute(S_STATUS, REPL_SEND_BULK)
		if nextseq, err = server.sendSnapshot(session); err != nil {
			stdlog.Printf("[S %s] snap send broken (%s)\n", remoteHost, err)
			return
		}
	}

	if nextseq < 0 {
		nextseq = 0
	}
	if nextseq < server.synclog.MinSeq() || nextseq > server.synclog.MaxSeq()+1 {
		stdlog.Printf("[S %s] seq %d not in (%d, %d), closed\n", remoteHost, nextseq, server.synclog.MinSeq(), server.synclog.MaxSeq())
		return errors.New("bad seq range")
	}

	// 如果整个同步过程s

	stdlog.Println("sync online ...")
	session.SetAttribute(S_STATUS, REPL_ONLINE)
	// 发送日志数据
	err = server.syncRunloop(session, nextseq)
	if err != nil {
		stdlog.Printf("[S %s] sync broken (%s)\n", remoteHost, err)
	}
	return
}

func (s *SyncLog) initSeq() {
	prefix := bytes.Join([][]byte{s.prefix, []byte(":id:")}, []byte(""))
	s.db.PrefixEnumerate(prefix, levelredis.IterForward, func(i int, key, value []byte, quit *bool) {
		s.minseq = s.splitSeqkey(key)
		*quit = true
	})
	s.db.PrefixEnumerate(prefix, levelredis.IterBackward, func(i int, key, value []byte, quit *bool) {
		s.seq = s.splitSeqkey(key)
		*quit = true
	})
	s.enabled = s.seq != -1
	if s.enabled {
		stdlog.Printf("synclog enabled, seq (%d, %d)\n", s.minseq, s.seq)
	}
}

// go run redis-proxy.go -p 1603 -master localhost:6379 -slave localhost:6389
func main() {
	runtime.GOMAXPROCS(8)
	// options
	opt := goredis_proxy.NewOptions()
	// flags
	version := flag.Bool("v", false, "print goredis-proxy version")
	flag.StringVar(&opt.Host, "h", "", "server host")
	flag.IntVar(&opt.Port, "p", 1602, "server port")
	flag.StringVar(&opt.MasterHost, "master", "", "master")
	flag.StringVar(&opt.SlaveHost, "slave", "", "slave")
	flag.StringVar(&opt.Mode, "mode", "rrw", "r/rr/rw/rrw, default rrw")
	flag.IntVar(&opt.PoolSize, "poolsize", 100, "pool for remote server")
	flag.Parse()

	if *version {
		fmt.Println("redis-proxy ", goredis_proxy.VERSION)
		return
	}

	if len(opt.MasterHost) == 0 || len(opt.SlaveHost) == 0 {
		stdlog.Println("bad master/slave")
		return
	}

	stdlog.Println("redis-proxy ", goredis_proxy.VERSION)
	stdlog.Printf("master:[%s], slave:[%s], mode:[%s], poolsize:[%d]\n", opt.MasterHost, opt.SlaveHost, opt.Mode, opt.PoolSize)
	stdlog.Println("listen", opt.Addr())

	// start
	server := goredis_proxy.NewProxy(opt)
	err := server.Init()
	if err != nil {
		panic(err)
	}
	err = server.Listen(opt.Addr())
	if err != nil {
		panic(err)
	}
}

func (s *SlaveClient) recvRdb() (err error) {
	session := s.src
	var f *os.File
	f, err = os.OpenFile(s.rdbfilename(), os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.ModePerm)
	if err != nil {
		return
	}
	stdlog.Printf("[M %s] create rdb:%s\n", session.RemoteAddr(), s.rdbfilename())

	session.ReadByte()
	var size int64
	size, err = session.ReadInt64()
	if err != nil {
		return
	}
	s.RdbSizeCallback(size)

	// read
	w := bufio.NewWriter(f)
	// var written int64
	_, err = iotool.RateLimitCopy(w, io.LimitReader(session, size), s.pullrate, func(written int64, rate int) {
		s.RdbRecvProcessCallback(written, rate)
	})
	if err != nil {
		return
	}
	w.Flush()
	f.Seek(0, 0)
	// 不阻塞进行接收command
	go func() {
		s.RdbRecvFinishCallback(bufio.NewReader(f))
		filename := f.Name()
		f.Close()
		os.Remove(filename)
	}()
	return
}