Golang loglib.Error函数代码示例

/*
* 行数为非负值表示已tail的行数
* 行数为负值则都将从文件末尾开始
* 自动保存的行数只可能是bufSize的倍数或者文件总行数
 */
func getLineRecord(path string) (line int, fname string) {
	fin, err := os.Open(path)
	if err != nil {
		_, f, l, _ := runtime.Caller(0)
		loglib.Error(fmt.Sprintf("%s:%d open line record `%s` error\n", f, l, path))
		return -1, "" //从最后开始读
	}
	var txt string
	var lineStr = ""
	//只读第一行
	scanner := bufio.NewScanner(fin)
	for scanner.Scan() {
		txt = strings.Trim(scanner.Text(), " ")
		break
	}
	fin.Close()
	parts := strings.Split(txt, " ")
	if len(parts) == 2 {
		fname = parts[0]
		lineStr = parts[1]
	} else {
		lineStr = parts[0]
	}
	line, err = strconv.Atoi(lineStr)
	if err != nil {
		loglib.Error("convert line record error:" + err.Error())
		line = -1
	}
	return line, fname
}

//保存footprint
func (t *TcpReceiver) saveFootPrint() {
	vbytes, err := json.Marshal(t.footPrint)
	if err != nil {
		loglib.Error("marshal footprint error:" + err.Error())
		return
	}
	err = ioutil.WriteFile(t.footPrintFile, vbytes, 0664)
	if err == nil {
		loglib.Info("save footprint success !")
	} else {
		loglib.Error("save footprint error:" + err.Error())
	}
}

func saveLineRecord(path string, fname string, lineNum int) {
	fout, err := os.Create(path)
	defer fout.Close()
	if err != nil {
		loglib.Error("save line record error: " + err.Error())
		return
	}
	_, err = fmt.Fprintf(fout, "%s %d", fname, lineNum)
	if err != nil {
		loglib.Error("Write line record error" + err.Error())
		return
	}
	loglib.Info("save line record success!")
}

func (this *MongoDbOutputer) Start() {
	wg := &sync.WaitGroup{}
	defer func() {
		if err := recover(); err != nil {
			loglib.Error(fmt.Sprintf("mongodb outputer panic:%v", err))
		}
		if this.session != nil {
			this.session.Close()
		}
		this.wq.AllDone()
	}()

	this.reloadFileCache()

	go this.reConnMongoDb()

	wg.Add(this.savers)

	for i := 0; i < this.savers; i++ {
		go this.runParse(i, wg)
	}

	nRetry := this.savers/6 + 1
	wg.Add(nRetry)
	for i := 0; i < nRetry; i++ {
		go this.retrySave(wg, i)
	}

	wg.Wait()

}

func (this *Tailler) Tailling(receiveChan chan map[string]string) {
	if this.currFile == "" {
		//兼容老格式，老格式无文件路径
		this.currFile = this.getLogFileByTime(time.Now())
	}
	var err error
	this.fileHour, err = this.getTimeFromLogName(this.currFile)
	if err != nil {
		loglib.Error("can't get time from current log file:" + this.currFile + "error:" + err.Error())
		os.Exit(1)
	}
	isQuit := false
	for time.Since(this.fileHour).Hours() >= 1 {
		//说明重启时已经跟记录行号时不属于同一个小时了
		isQuit = this.taillingPrevious(this.currFile, this.lineNum, this.fileHour.Format(this.hourStrFmt), receiveChan)
		if isQuit {
			break
		}
		//继续下一个小时
		this.fileHour = this.fileHour.Add(time.Hour)
		this.currFile = this.getLogFileByTime(this.fileHour)
		this.lineNum = 0
	}
	if !isQuit {
		//处理当前这个小时
		this.taillingCurrent(receiveChan)
	}
	close(receiveChan)
	this.wq.AllDone()
}

func (lr *LogReceiver) Run() {
	l, err := net.Listen("tcp", fmt.Sprintf(":%d", lr.port))
	if err != nil {
		loglib.Error("[log receiver] " + err.Error())
		return
	}
	defer l.Close()
	for {
		conn, err := l.Accept()
		if err != nil {
			loglib.Error("[log receiver] " + err.Error())
			return
		}
		go lr.handleConnection(conn)
	}

}

func (f *fileOutputer) extract(bp *bytes.Buffer) {
	buf := make([]byte, 4)
	bp.Read(buf)

	l, _ := binary.Uvarint(buf)
	headerLen := int(l)
	//get pack header
	buf = make([]byte, headerLen)
	bp.Read(buf)
	header := tcp_pack.ParseHeader(buf)

	r, err := zlib.NewReader(bp)
	if err != nil {
		loglib.Error("zlib reader Error: " + err.Error())
	} else {
		lines, _ := strconv.Atoi(header["lines"])
		done := false
		if header["done"] == "1" {
			done = true
		}
		f.ic.Add(header["ip"], header["hour"], header["id"], lines, done)

		writerKey := header["ip"] + "_" + header["hour"]
		fout := f.getWriter(f.writers, f.dataDir, writerKey)

		//一头一尾写头信息，节省硬盘
		buf = append(buf, '\n')
		//fout.Write(buf)
		nn, err := io.Copy(fout, r)
		if err != nil {
			loglib.Warning(fmt.Sprintf("save %s_%s_%s error:%s, saved:%d", header["ip"], header["hour"], header["id"], err, nn))
		}
		//fout.Write(buf)

		//单独存一份header便于查数
		fout = f.getWriter(f.headerWriters, f.headerDir, writerKey)
		n, err := fout.Write(buf)
		if err != nil {
			loglib.Info(fmt.Sprintf("writer header %s %d %s", writerKey, n, err.Error()))
		}

		if done || time.Now().Unix() > f.checkTime.Unix() {
			hourFinish, _ := f.ic.Check()
			for ip, hours := range hourFinish {
				for _, hour := range hours {
					writerKey = ip + "_" + hour
				}
			}
			f.closeWriters(f.writers)
			f.closeWriters(f.headerWriters)
			f.checkTime.Add(2 * time.Minute)
		}

		r.Close()
	}
}

func (t *TcpReceiver) loadFootPrint(fname string) map[string]PackAppear {
	fp := make(map[string]PackAppear)
	if lib.FileExists(fname) {
		vbytes, err := ioutil.ReadFile(fname)
		if err != nil {
			loglib.Error("read footprint file error:" + err.Error())
		} else {
			err = json.Unmarshal(vbytes, &fp)
			if err != nil {
				loglib.Error("unmarshal footprint error:" + err.Error())
			} else {
				loglib.Info("load footprint success !")
			}
		}
	} else {
		loglib.Warning("footprint file " + fname + " not found!")
	}
	return fp
}

//init conn list from addrMap
func (sc *SingleConnection) initConnection() {
	newConn, err := createSingleConnection(sc.currentAddr)
	if err != nil {
		loglib.Error("init err:" + err.Error())
	} else {
		sc.conn = newConn
	}

	lib.CheckError(err)
}

func (e *etlOutputer) Start() {
	defer func() {
		if err := recover(); err != nil {
			loglib.Error(fmt.Sprintf("etl outputer panic:%v", err))
		}

		e.wq.AllDone()
	}()

	spiderList, _ := e.config["spider_list"]
	colsFile, _ := e.config["columns_file"]
	hostsList, _ := e.config["hosts_white_list"]
	ipBlackList, _ := e.config["ip_black_list"]

	if colsFile != "" {
		e.runEtl(spiderList, colsFile, hostsList, ipBlackList)
	} else {
		loglib.Error("[error] miss columns map file!")
	}
}

func initMongoDbSession(mongosAddr string) *mgo.Session {
	session, err := mgo.Dial(mongosAddr)
	if err != nil {
		loglib.Error(fmt.Sprintf("init mongodb session error:%v", err))
		return nil
	}

	session.SetMode(mgo.Monotonic, true) //设置read preference
	session.SetSafe(&mgo.Safe{W: 2})     //设置write concern
	return session
}

func (this *MongoDbOutputer) bulkSaveBson(coll *mgo.Collection, docs ...interface{}) (err error) {
	if coll != nil {
		err = coll.Insert(docs...)
		if err != nil {
			tmp := make([]string, 0)
			for _, doc := range docs {
				m, _ := doc.(bson.M)
				tid, _ := m[this.transactionIdKey].(string)
				tmp = append(tmp, tid)
			}
			tids := strings.Join(tmp, ",")
			loglib.Error(fmt.Sprintf("save %d bsons [%s] error:%v", len(docs), tids, err))
		}
	} else {
		err = errors.New("bulk: collection is nil")
		loglib.Error(fmt.Sprintf("save bsons error:%v", err))

	}
	return
}

func (this *MongoDbOutputer) upsertBson(coll *mgo.Collection, selector interface{}, doc interface{}) (info *mgo.ChangeInfo, err error) {
	m, _ := selector.(bson.M)
	tid, _ := m[this.transactionIdKey].(string)

	if coll != nil {
		info, err = coll.Upsert(selector, doc)

		if err != nil {
			loglib.Error(fmt.Sprintf("save bson [%s] error:%v", tid, err))
		} else {
			if info.Updated > 0 {
				loglib.Info(fmt.Sprintf("bson [%s] updated", tid))
			}
		}
	} else {
		info = &mgo.ChangeInfo{}
		err = errors.New("upsert: collection is nil")
		loglib.Error(fmt.Sprintf("save bson [%s] error:%v", tid, err))
	}
	return
}

func (this *Tailler) GetTotalLines(fname string) int {
	cmd := exec.Command("/bin/sh", "-c", `wc -l `+fname+` | awk '{print $1}'`)
	out, err := cmd.Output()
	if err == nil {
		n, err := strconv.Atoi(strings.Trim(string(out), "\n"))
		if err != nil {
			loglib.Error("trans total lines " + string(out) + " error: " + err.Error())
		}
		return n
	}
	return 0
}

func NewTailler(config map[string]string) *Tailler {
	val, ok := config[logFileKey]
	if !ok || val == "" {
		loglib.Error("config need log_file!")
		os.Exit(1)
	}
	logPath := val
	val, ok = config[recordFileKey]
	if !ok || val == "" {
		config[recordFileKey] = getRecordPath()
	}
	lineNum, fname := getLineRecord(config[recordFileKey])
	goFmt, nLT := extractTimeFmt(logPath)
	if goFmt == "" {
		loglib.Error("log path has no time format!")
		os.Exit(1)
	}
	wq := lib.NewWaitQuit("tailler")
	bufSize, _ := strconv.Atoi(config["recv_buffer_size"])

	return &Tailler{logPath: logPath, nLT: nLT, currFile: fname, hourStrFmt: "2006010215", lineNum: lineNum, goFmt: goFmt, recordPath: config[recordFileKey], config: config, recvBufSize: bufSize, wq: wq}
}