Golang MapStr.EnsureTimestampField方法代码示例

//TODO: Check for Errors Here
func (jsonexpander *JSONExpander) Filter(event common.MapStr) (common.MapStr, error) {
	text := event["message"]
	text_string := text.(*string)
	logp.Debug("jsonexpander", "Attempting to expand: %v", event)

	if isJSONString(*text_string) {
		data := []byte(*text_string)
		err := json.Unmarshal(data, &event)
		if err != nil {
			logp.Err("jsonexpander", "Could not expand json data")
			return event, nil
		}
	} else {
		logp.Debug("jsonexpander", "Message does not appear to be JSON data: %s", text_string)
	}

	now := func() time.Time {
		t := time.Now()
		return t
	}

	event.EnsureTimestampField(now)

	logp.Debug("jsonexpander", "Final Event: %v", event)
	return event, nil
}

func (l *TailInput) doStuff(output chan common.MapStr) {
	now := func() time.Time {
		t := time.Now()
		return t
	}

	var line uint64 = 0
	var read_timeout = 30 * time.Second

	// open file
	// basic error handling, if we hit an error, log and return
	// this ends the currently running thread without impacting other threads
	f, err := os.Open(l.FileName)
	if err != nil {
		logp.Err("Error opening file " + err.Error())
		return
	}
	l.FileP = f

	// seek to end
	// for offset, we use the actual file offset
	// we initialize it to the end of the file at time of open
	l.offset, err = l.FileP.Seek(0, 2)
	if err != nil {
		logp.Err("Error seeking in file " + err.Error())
		return
	}
	l.LastOpen = time.Now()

	buffer := new(bytes.Buffer)
	reader := bufio.NewReader(l.FileP)

	for {
		l.CheckReopen()
		text, bytesread, err := readline(reader, buffer, read_timeout)
		if err != nil && err != io.EOF {
			// EOF errors are expected, since we are tailing the file
			logp.Err("Error reading file " + err.Error())
			return
		}

		if bytesread > 0 {
			l.offset += int64(bytesread)
			line++

			event := common.MapStr{}
			event["filename"] = l.FileName
			event["line"] = line
			event["message"] = text
			event["offset"] = l.offset
			event["type"] = l.Type

			event.EnsureTimestampField(now)
			event.EnsureCountField()

			logp.Debug("tailinput", "InputEvent: %v", event)
			output <- event // ship the new event downstream
		}
	}
}

func (l *PackagesInput) doStuff(output chan common.MapStr) {

	now := func() time.Time {
		t := time.Now()
		return t
	}

	// construct event and write it to channel
	event := common.MapStr{}

	//text := "null event"
	//event["message"] = &text

	event["message"] = "packages event"
	event["type"] = l.Type

	event.EnsureTimestampField(now)
	event.EnsureCountField()

	/////////////
	cmd := exec.Command("/bin/rpm", "-qa", "--queryformat", "%{NAME}:::%{VERSION}:::%{ARCH}##")
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		logp.Info("Error occurred")
		return
	}

	items := strings.Split(out.String(), "##")
	rpmList := make([]RPMPackage, 0)

	for _, line := range items {
		item := strings.Split(line, ":::")
		if len(item) < 3 {
			continue
		}

		pkg := RPMPackage{
			Name:    item[0],
			Version: item[1],
			Arch:    item[2],
		}

		rpmList = append(rpmList, pkg)
	}

	event["packages"] = rpmList
	output <- event

}

func (l *StdinInput) doStuff(output chan common.MapStr) {
	reader := bufio.NewReader(os.Stdin)
	buffer := new(bytes.Buffer)

	var source string = fmt.Sprintf("%s:%s", os.Getenv("REMOTE_HOST"), os.Getenv("REMOTE_PORT"))
	var ssl_client_dn string = os.Getenv("SSL_CLIENT_DN")
	var offset int64 = 0
	var line uint64 = 0
	var read_timeout = 10 * time.Second

	logp.Debug("stdinput", "Handling New Connection from %s", source)

	now := func() time.Time {
		t := time.Now()
		return t
	}

	for {
		text, bytesread, err := l.readline(reader, buffer, read_timeout)

		if err != nil {
			logp.Info("Unexpected state reading from %v; error: %s\n", os.Getenv("SSL_CLIENT_DN"), err)
			return
		}

		logp.Debug("stdinputlines", "New Line: %s", &text)

		line++

		event := common.MapStr{}
		event["ssl_client_dn"] = &ssl_client_dn
		event["source"] = &source
		event["offset"] = offset
		event["line"] = line
		event["message"] = text
		event["type"] = l.Type

		event.EnsureTimestampField(now)
		event.EnsureCountField()

		offset += int64(bytesread)

		logp.Debug("stdinput", "InputEvent: %v", event)
		output <- event // ship the new event downstream
		os.Stdout.Write([]byte("OK"))
	}
	logp.Debug("stdinput", "Closed Connection from %s", source)
}

func (l *TcpInput) handleConn(client net.Conn, output chan common.MapStr) {
	reader := bufio.NewReader(client)
	buffer := new(bytes.Buffer)

	var source string = client.RemoteAddr().String()
	var offset int64 = 0
	var line uint64 = 0
	var read_timeout = 10 * time.Second

	logp.Debug("tcpinput", "Handling New Connection from %s", source)

	now := func() time.Time {
		t := time.Now()
		return t
	}

	for {
		text, bytesread, err := l.readline(reader, buffer, read_timeout)

		if err != nil {
			logp.Info("Unexpected state reading from %v; error: %s\n", client.RemoteAddr().String, err)
			return
		}

		logp.Debug("tcpinputlines", "New Line: %s", &text)

		line++

		event := common.MapStr{}
		event["source"] = &source
		event["offset"] = offset
		event["line"] = line
		event["message"] = text
		event["type"] = l.Type

		event.EnsureTimestampField(now)
		event.EnsureCountField()

		offset += int64(bytesread)

		logp.Debug("tcpinput", "InputEvent: %v", event)
		output <- event // ship the new event downstream
		client.Write([]byte("OK"))
	}
	logp.Debug("tcpinput", "Closed Connection from %s", source)
}

func (l *NullInput) doStuff(output chan common.MapStr) {
	now := func() time.Time {
		t := time.Now()
		return t
	}

	// construct event and write it to channel
	event := common.MapStr{}

	text := "null event"
	event["message"] = &text
	event["type"] = l.Type

	event.EnsureTimestampField(now)
	event.EnsureCountField()

	output <- event

}

func (l *UdpInput) handlePacket(buffer []byte, rlen int, count int, source *net.UDPAddr, output chan common.MapStr) {
	now := func() time.Time {
		t := time.Now()
		return t
	}

	text := string(buffer[0:rlen])

	logp.Debug("udpinputlines", "New Line: %s", &text)

	event := common.MapStr{}
	event["source"] = &source
	event["offset"] = rlen
	event["line"] = count
	event["message"] = &text
	event["type"] = l.Type

	event.EnsureTimestampField(now)
	event.EnsureCountField()

	logp.Debug("udpinput", "InputEvent: %v", event)
	output <- event // ship the new event downstream
}

func scanProcs(output chan common.MapStr) {
	now := func() time.Time {
		t := time.Now()
		return t
	}

	if !pathExists(procfsdir) {
		return
	}
	ds, err := ioutil.ReadDir(procfsdir)
	if err != nil {
		return
	}

	event := common.MapStr{}
	processes := common.MapStr{}
	proc_detail := common.MapStr{}

	// get all numeric entries
	for _, d := range ds {
		n := d.Name()
		if isNumeric(n) {
			processes[n] = getCmdline(n)
			proc_detail[n] = getProcDetail(n)
		}
	}

	text := "process report"
	event["message"] = &text
	event["data"] = processes
	event["data_detail"] = proc_detail
	event["type"] = "report"

	event.EnsureTimestampField(now)
	event.EnsureCountField()
	output <- event
}

func (l *RedisInput) handleConn(server redis.Conn, output chan common.MapStr, key string) (uint64, error) {
	event_slice, offset, timestamp, tag_string, err := l.readKey(server, key)

	if err != nil {
		logp.Err("an error reading %s: %s\n", key, err)
	}

	now := func() time.Time {
		t := time.Now()
		return t
	}

	parsed_tags := strings.Fields(tag_string)
	tags := make(map[string]string)

	for _, v := range parsed_tags {
		tag := strings.Split(v, "=")
		tags[tag[0]] = tag[1]
	}

	t := time.Unix(timestamp, 0)
	data, _ := json.Marshal(t)

	// check that metric_name exists and metric_value
	metric_event := common.MapStr{}
	var value string
	for _, event := range event_slice {
		switch vartype := event["metric_value"].(type) {
		case int:
			logp.Debug("redisinput", fmt.Sprintf("vartype is %s", vartype))
			value = fmt.Sprintf("%v", event["metric_value"].(int))
		case float64:
			logp.Debug("redisinput", fmt.Sprintf("vartype is %s", vartype))
			value = fmt.Sprintf("%v", event["metric_value"].(float64))
		case string:
			logp.Debug("redisinput", fmt.Sprintf("vartype is %s", vartype))
			value = fmt.Sprintf("%v", event["metric_value"].(string))
		}
		metric_event[event["metric_name"].(string)] = value
	}

	event := common.MapStr{}
	event["name"] = strings.TrimSpace(key)
	event["offset"] = offset
	event["count"] = len(event_slice)
	event["type"] = strings.TrimSpace(l.Type)
	event["tags"] = tags
	event["metrics"] = metric_event
	event["timestamp"] = string(data)

	event.EnsureTimestampField(now)
	event.EnsureCountField()

	logp.Debug("redisinputlines", "event: %v", event)
	if event_slice != nil {
		output <- event // ship the new event downstream
	}

	logp.Debug("redisinput", "Finished reading from %s", key)
	return uint64(len(event_slice)), nil
}

func (l *SyslogInput) Run(output chan common.MapStr) error {
	logp.Debug("sysloginput", "Running Syslog Input")
	logp.Debug("sysloginput", "Listening on %d", l.Port)

	listen := fmt.Sprintf("0.0.0.0:%d", l.Port)

	channel := make(syslog.LogPartsChannel)
	handler := syslog.NewChannelHandler(channel)

	server := syslog.NewServer()
	server.SetFormat(syslog.Automatic)
	server.SetHandler(handler)
	err := server.ListenUDP(listen)
	if err != nil {
		logp.Err("couldn't start ListenUDP: " + err.Error())
	}
	err = server.ListenTCP(listen)
	if err != nil {
		logp.Err("couldn't start ListenTCP: " + err.Error())
	}
	err = server.Boot()
	if err != nil {
		logp.Err("couldn't start server.Boot(): " + err.Error())
	}

	go func(channel syslog.LogPartsChannel, output chan common.MapStr) {
		var line uint64 = 0

		now := func() time.Time {
			t := time.Now()
			return t
		}

		for logParts := range channel {
			logp.Debug("sysloginput", "InputEvent: %v", logParts)

			line++
			event := common.MapStr{}
			event["line"] = line
			event["type"] = l.Type

			for k, v := range logParts {
				event[k] = v
			}

			event["source"] = event["client"].(string)

			if event["message"] != nil {
				message := event["message"].(string)
				event["message"] = &message
			} else if event["content"] != nil {
				message := event["content"].(string)
				event["message"] = &message
			}

			// This syslog parser uses the standard name "tag"
			// which is usually the program that wrote it.
			// The logstash syslog_pri puts "program" for this field.
			if event["tag"] != nil {
				program := event["tag"].(string)
				event["program"] = &program
			}

			event.EnsureTimestampField(now)
			event.EnsureCountField()

			logp.Debug("sysloginput", "Output Event: %v", event)
			output <- event // ship the new event downstream
		}
	}(channel, output)

	return nil
}