Golang logs.Log函数代码示例

//获取所有任务的任务列表
func FetchTaskList() (int, *list.List) {
	var stmtIns *sql.Stmt
	var err error
	stmtIns, err = db.Prepare("select task_id, create_time from rdb_parse_task")

	if err != nil {
		logs.Log("fetch task list error in dbutil db.Prepare : " + err.Error())
		return -1, list.New()
	}

	defer stmtIns.Close()

	rows, errs := stmtIns.Query()

	if errs != nil {
		logs.Log("fetch task list error in dbutil stmtIns.Query : " + errs.Error())
		return -2, list.New()
	}

	taskList := list.New()

	var taskId string
	var createTime string
	for rows.Next() {
		var task Task
		rows.Scan(&taskId, &createTime)
		task.TaskId = taskId
		task.CreateTime = createTime
		taskList.PushBack(task)
	}
	return 1, taskList
}

//根据taskId和type获取分析结果
func FetchResult(taskId string, types string, limit int, offset int) []string {
	result := make([]string, 0)
	stmtIns, err := db.Prepare("select content from rdb_anly_result where task_id = ? and type = ? order by id desc limit ? offset ? ")

	if err != nil {
		logs.Log("fetch result error in dbutil db.Prepare!" + " : " + err.Error())
		return result
	}

	defer stmtIns.Close()

	rows, errs := stmtIns.Query(taskId, types, limit, offset)
	if errs != nil {
		logs.Log("fetch result error in dbutil stmtIns.Query!" + " : " + errs.Error())
		return result
	}

	var content string

	for rows.Next() {
		rows.Scan(&content)
		result = append(result, content)
	}
	return result
}

//删除任务，以及任务的分析结果
func DeleteTask(list *list.List) bool {

	stmtInsTask, err := db.Prepare("delete from rdb_parse_task where task_id = ?")

	if err != nil {
		logs.Log("delete task stmtInsTask error in dbutil : " + err.Error())
		return false
	}

	defer stmtInsTask.Close()

	stmtResult, err := db.Prepare("delete from rdb_anly_result where task_id = ?")
	if err != nil {
		logs.Log("delete dask stmtResult error in dbutil : " + err.Error())
		return false
	}

	defer stmtResult.Close()

	for e := list.Front(); e != nil; e = e.Next() {
		taskId, ok := e.Value.(string)
		if ok {
			stmtInsTask.Exec(taskId)
			stmtResult.Exec(taskId)
		} else {
			logs.Log("type error!")
		}
	}
	return true
}

//管道读取
func pipelineAnalyze(inputFile string, param *Parameters) {
	file, err := os.Open(inputFile)

	if err != nil && err != io.EOF {
		logs.Log("file open error in parseRdb pipelineAnalyze : " + err.Error())
		return
	}

	defer file.Close()

	//首先读取一定数量的字节
	buff := make([]byte, (*param).SliceContent)

	n, err := file.Read(buff)
	if err != nil && err != io.EOF {
		logs.Log("file read error in parseRdb pipelineAnalyze : " + err.Error())
		return
	}

	//解析
	var pbuff = &buff
	if n != 0 {
		(*param).OffsetBytes = (*param).OffsetBytes + int64(n)
		parseRedisRdb(pbuff, inputFile, param)
		(*param).FixedInfo = true
	}
	//为了防止文件没有全部读取，再次循环读取,理论上这个地方应该用不到
	for {
		n, err := file.ReadAt(buff, (*param).OffsetBytes)

		if err != nil && err != io.EOF {
			logs.Log("file read error in parseRdb pipelineAnalyze for loop : " + err.Error())
			return
		}

		defer file.Close()

		if n == 0 {
			break
		} else {
			(*param).SliceStart = 0
			(*param).SliceEnd = (*param).SliceStart + 1
			(*param).OffsetBytes = (*param).OffsetBytes + int64(n)
			parseRedisRdb(pbuff, inputFile, param)
			(*param).FixedInfo = true
		}
	}
}

/**
这个是解析的入口
pwflag 表示是打印还是写入到文件
0 打印
1 写入到文件
2 写入到数据库
filename 表示要解析的文件路径
*/
func Start(inputFile string, filterLength int, filterKey string) Parameters {

	//读取管道的大小
	conf := config.SetConfig("important.properties")
	sliceContent, _ := strconv.ParseUint(conf.GetValue("pipeline", "size"), 10, 64)
	if sliceContent < 16 {
		sliceContent = 16
		logs.Log("sliceContent < 16")
	}

	var heapIni bool = false

	var param Parameters
	param.Success = false
	param.SliceStart = 0
	param.SliceEnd = 0
	param.Step = []uint64{5, 4}
	param.FileFlag = false
	param.SliceContent = sliceContent
	param.FixedInfo = false
	param.OffsetBytes = 0
	param.Dbnum = -1
	param.ResultMap = make(map[string]int)
	param.ValueFilterLength = filterLength
	param.FilterKey = filterKey
	param.ValueLengthFilterList = list.New()
	param.KeyFilterMap = make(map[string]int)
	param.Heap = &sorting.Heap{}
	param.HeapInit = &heapIni
	pipelineAnalyze(inputFile, &param)

	return param
}

//更新任务状态
func UpdateTaskStatus(status int, taskId string) bool {
	stmtIns, err := db.Prepare("update rdb_parse_task set status = ? where task_id = ?")

	if err != nil {
		logs.Log("update task status error in dbutil db.Prepare : " + err.Error())
		return false
	}
	defer stmtIns.Close()

	_, errs := stmtIns.Exec(status, taskId)
	if errs != nil {
		logs.Log("update task status error in dbutil stmtIns.Exec : " + errs.Error())
		return false
	}
	return true
}

//存储分析结果
func Store(content, types, taskId string) {

	stmtIns, err := db.Prepare("insert into rdb_anly_result(content, type, task_id, create_time) values (?,?,?,now())")

	if err != nil {
		logs.Log("store error in dbutil db.Prepare : " + err.Error())
		return
	}
	defer stmtIns.Close()

	_, errs := stmtIns.Exec(content, types, taskId)

	if errs != nil {
		logs.Log("store error in dbutil stmtIns.Exec : " + errs.Error())
		return
	}
}

//向任务表中插入任务
func InsertTask(host, port, taskId string, filterLength, priority int, filter_key string) bool {

	stmtIns, err := db.Prepare("insert into rdb_parse_task (host, port, filter_length, filter_key, task_id, priority, status, create_time) values (?,?,?,?,?,?,0,now())")

	if err != nil {
		logs.Log("insert task error in dbutil db.Prepare : " + err.Error())
		return false
	}
	defer stmtIns.Close()

	_, errs := stmtIns.Exec(host, port, filterLength, filter_key, taskId, priority)
	if errs != nil {
		logs.Log("insert task error in dbutil stmtIns.Exec : " + errs.Error())
		return false
	}
	return true
}

//解析程序
func parseTask(host string, port string, filterLength int, filterKey string, taskId string) {

	//获得机器的ip地址
	ip := utils.GetIpAddress()
	if ip == "unknown" {
		ip = utils.GetIpAddress2()
	}
	logs.Log("machine : " + ip + " processing task : " + taskId)
	//获取rdb文件
	rdbFile := rdbLocation + taskId + ".rdb"

	//如果文件存在则删除
	isExist := exists(rdbFile)
	if isExist == true {
		err := os.Remove(rdbFile)
		if err != nil {
			logs.Log("parseTask remove rdbFile error : " + err.Error())
			return
		}
	}

	flag := syncRdb(host, port, cliLocation, rdbFile, taskId)
	if flag == 1 {
		//还要判断解析是否成功了
		status := start(rdbFile, taskId, filterLength, filterKey)
		if status == true {
			dbutil.UpdateTaskStatus(1, taskId)
		} else {
			dbutil.UpdateTaskStatus(3, taskId)
		}
		//删除文件
		dropFile(rdbFile, taskId)
	} else {
		switch flag {
		case 2: //同步进程失效，文件大小无变化
			logs.Log("task " + taskId + " sync process error, file size no change!")
			dbutil.UpdateTaskStatus(2, taskId)
		case 4: //获得redis密码错误
			dbutil.UpdateTaskStatus(4, taskId)
			logs.Log("task " + taskId + " get remote redis server login password error!")
		}
		//删除文件
		dropFile(rdbFile, taskId)
	}
}

/**
net 环境下能够获得访问网络的ip
*/
func GetIpAddress() string {
	conn, err := net.Dial("udp", "www.baidu.com:80")
	if err != nil {
		logs.Log(err.Error())
		return "unknown"
	}
	defer conn.Close()
	return strings.Split(conn.LocalAddr().String(), ":")[0]
}

//解析操作完成之后删除文件
func dropFile(file, taskId string) {
	//首先判断存在文件吗
	isExist := exists(file)
	if isExist {
		err := os.Remove(file)
		if err != nil {
			logs.Log("task " + taskId + " remove file error : " + err.Error() + " --- " + file)
		}
	}
}

//根据任务id获取单个任务
func FetchTaskById(task_id string) (int, *list.List) {
	stmtIns, err := db.Prepare("select host, port, filter_length, filter_key, task_id, priority, status, create_time from rdb_parse_task where task_id = ?")

	if err != nil {
		logs.Log("fetch task by id error in dbutil db.Prepare : " + err.Error())
		return -1, list.New()
	}

	defer stmtIns.Close()

	rows, errs := stmtIns.Query(task_id)

	if errs != nil {
		logs.Log("fetch task by id error in dbutil stmtIns.Query : " + errs.Error())
		return -2, list.New()
	}

	taskList := list.New()

	var host string
	var port string
	var filterLength int
	var filterKey string
	var taskId string
	var priority int
	var status int
	var createTime string
	for rows.Next() {
		var task Task
		rows.Scan(&host, &port, &filterLength, &filterKey, &taskId, &priority, &status, &createTime)
		task.Host = host
		task.Port = port
		task.FilterLength = filterLength
		task.TaskId = taskId
		task.Priority = priority
		task.Status = status
		task.CreateTime = createTime
		task.FilterKey = filterKey
		taskList.PushBack(task)
	}
	return 1, taskList
}

//存储堆排序结果
func StoreSortingResult(h *sorting.Heap, taskId string) {
	for h.Len() > 0 {
		element := heap.Pop(h).(sorting.Element)
		b, err := json.Marshal(element)
		if err != nil {
			logs.Log("json marshal error in StoreValueTypeStat : " + err.Error())
			return
		}
		dbutil.Store(string(b), "2", taskId)
	}
}

//为了部署多个实例的需要，修改了startParse
func startParse() {

	//每次只取一个任务，这样保证当存在多个服务时，每个机器都能获得任务
	logs.Log("starting parse!")
	for {
		mark, task := dbutil.FetchNewTask()

		if mark == 1 {
			if task.TaskId == "" {
				time.Sleep(time.Second * 10)
				continue
			} else {
				parseTask(task.Host, task.Port, task.FilterLength, task.FilterKey, task.TaskId)
			}
		} else if mark == -5 {
			logs.Log("task " + task.TaskId + " processing by another machine! ")
		} else {
			logs.Log("fetch task error!")
		}
	}
}

func StoreValueLengthFilter(list *list.List, fileId string) {

	for e := list.Front(); e != nil; e = e.Next() {
		str, ok := e.Value.(string)
		if ok {
			dbutil.Store(str, "2", fileId)
		} else {
			logs.Log("list element error in StoreValueLengthFilter list element is not string!")
			continue
		}
	}
}