Golang message.NewField函數代碼示例

func getTestMessageWithFunnyFields() *message.Message {
	field, _ := message.NewField(`"foo`, "bar\n", "")
	field1, _ := message.NewField(`"number`, 64, "")
	field2, _ := message.NewField("\xa3", "\xa3", "")
	field3, _ := message.NewField("idField", "1234", "")

	msg := &message.Message{}
	msg.SetType("TEST")
	loc, _ := time.LoadLocation("UTC")
	t, _ := time.ParseInLocation("2006-01-02T15:04:05.000Z", "2013-07-16T15:49:05.070Z",
		loc)
	msg.SetTimestamp(t.UnixNano())
	msg.SetUuid(uuid.Parse("87cf1ac2-e810-4ddf-a02d-a5ce44d13a85"))
	msg.SetLogger("GoSpec")
	msg.SetSeverity(int32(6))
	msg.SetPayload("Test Payload")
	msg.SetEnvVersion("0.8")
	msg.SetPid(14098)
	msg.SetHostname("hostname")
	msg.AddField(field)
	msg.AddField(field1)
	msg.AddField(field2)
	msg.AddField(field3)

	return msg
}

func (s *SandboxDecoder) SetDecoderRunner(dr DecoderRunner) {
	s.sb.InjectMessage(func(payload, payload_type, payload_name string) int {
		if len(payload_type) == 0 { // heka protobuf message
			t := s.pack.Message.GetType()
			h := s.pack.Message.GetHostname()
			l := s.pack.Message.GetLogger()
			if nil != proto.Unmarshal([]byte(payload), s.pack.Message) {
				return 1
			}
			if s.pack.Message.GetType() == "" {
				s.pack.Message.SetType(t)
			}
			if s.pack.Message.GetHostname() == "" {
				s.pack.Message.SetHostname(h)
			}
			if s.pack.Message.GetLogger() == "" {
				s.pack.Message.SetLogger(l)
			}
		} else {
			s.pack.Message.SetPayload(payload)
			ptype, _ := message.NewField("payload_type", payload_type, "file-extension")
			s.pack.Message.AddField(ptype)
			pname, _ := message.NewField("payload_name", payload_name, "")
			s.pack.Message.AddField(pname)
		}
		return 0
	})
}

func (input *FilePollingInput) Run(runner pipeline.InputRunner,
	helper pipeline.PluginHelper) error {

	var (
		data    []byte
		pack    *pipeline.PipelinePack
		dRunner pipeline.DecoderRunner
		ok      bool
		err     error
	)

	if input.DecoderName != "" {
		if dRunner, ok = helper.DecoderRunner(input.DecoderName,
			fmt.Sprintf("%s-%s", runner.Name(), input.DecoderName)); !ok {
			return fmt.Errorf("Decoder not found: %s", input.DecoderName)
		}
		input.decoderChan = dRunner.InChan()
	}
	input.runner = runner

	hostname := helper.PipelineConfig().Hostname()
	packSupply := runner.InChan()
	tickChan := runner.Ticker()

	for {
		select {
		case <-input.stop:
			return nil
		case <-tickChan:
		}

		data, err = ioutil.ReadFile(input.FilePath)
		if err != nil {
			runner.LogError(fmt.Errorf("Error reading file: %s", err))
			continue
		}

		pack = <-packSupply
		pack.Message.SetUuid(uuid.NewRandom())
		pack.Message.SetTimestamp(time.Now().UnixNano())
		pack.Message.SetType("heka.file.polling")
		pack.Message.SetHostname(hostname)
		pack.Message.SetPayload(string(data))
		if field, err := message.NewField("TickerInterval", int(input.TickerInterval), ""); err != nil {
			runner.LogError(err)
		} else {
			pack.Message.AddField(field)
		}
		if field, err := message.NewField("FilePath", input.FilePath, ""); err != nil {
			runner.LogError(err)
		} else {
			pack.Message.AddField(field)
		}
		input.sendPack(pack)
	}

	return nil
}

func main() {
	configFile := flag.String("config", "sbmgr.toml", "Sandbox manager configuration file")
	scriptFile := flag.String("script", "xyz.lua", "Sandbox script file")
	scriptConfig := flag.String("scriptconfig", "xyz.toml", "Sandbox script configuration file")
	filterName := flag.String("filtername", "filter", "Sandbox filter name (used on unload)")
	action := flag.String("action", "load", "Sandbox manager action")
	flag.Parse()

	var config SbmgrConfig
	if _, err := toml.DecodeFile(*configFile, &config); err != nil {
		log.Printf("Error decoding config file: %s", err)
		return
	}
	sender, err := client.NewNetworkSender("tcp", config.IpAddress)
	if err != nil {
		log.Fatalf("Error creating sender: %s\n", err.Error())
	}
	encoder := client.NewProtobufEncoder(&config.Signer)
	manager := client.NewClient(sender, encoder)

	hostname, _ := os.Hostname()
	msg := &message.Message{}
	msg.SetType("heka.control.sandbox")
	msg.SetTimestamp(time.Now().UnixNano())
	msg.SetUuid(uuid.NewRandom())
	msg.SetHostname(hostname)

	switch *action {
	case "load":
		code, err := ioutil.ReadFile(*scriptFile)
		if err != nil {
			log.Printf("Error reading scriptFile: %s\n", err.Error())
			return
		}
		msg.SetPayload(string(code))
		conf, err := ioutil.ReadFile(*scriptConfig)
		if err != nil {
			log.Printf("Error reading scriptConfig: %s\n", err.Error())
			return
		}
		f, _ := message.NewField("config", string(conf), "toml")
		msg.AddField(f)
	case "unload":
		f, _ := message.NewField("name", *filterName, "")
		msg.AddField(f)
	default:
		log.Printf("Invalid action: %s", *action)
	}

	f1, _ := message.NewField("action", *action, "")
	msg.AddField(f1)
	err = manager.SendMessage(msg)
	if err != nil {
		log.Printf("Error sending message: %s\n", err.Error())
	}
}

func getSentryPack() (pack *pipeline.PipelinePack) {
	recycleChan := make(chan *pipeline.PipelinePack, 1)
	pack = pipeline.NewPipelinePack(recycleChan)
	pack.Message.SetType("sentry")
	fTimeStamp, _ := message.NewField("epoch_timestamp", EPOCH_TS, "utc-seconds")
	fDsn, _ := message.NewField("dsn", DSN, "uri")
	pack.Message.AddField(fTimeStamp)
	pack.Message.AddField(fDsn)
	pack.Message.SetPayload(PAYLOAD)
	pack.Decoded = true
	return
}

func (hli *HttpListenInput) RequestHandler(w http.ResponseWriter, req *http.Request) {

	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		fmt.Errorf("[HttpListenInput] Read HTTP request body fail: %s\n", err.Error())
	}
	req.Body.Close()

	unEscapedBody, _ := url.QueryUnescape(string(body))

	pack := <-hli.ir.InChan()
	pack.Message.SetUuid(uuid.NewRandom())
	pack.Message.SetTimestamp(time.Now().UnixNano())
	pack.Message.SetType("heka.httpdata.request")
	pack.Message.SetLogger(hli.ir.Name())
	pack.Message.SetHostname(req.RemoteAddr)
	pack.Message.SetPid(int32(os.Getpid()))
	pack.Message.SetSeverity(int32(6))
	pack.Message.SetPayload(unEscapedBody)
	if field, err := message.NewField("Protocol", req.Proto, ""); err == nil {
		pack.Message.AddField(field)
	} else {
		hli.ir.LogError(fmt.Errorf("can't add field: %s", err))
	}
	if field, err := message.NewField("UserAgent", req.UserAgent(), ""); err == nil {
		pack.Message.AddField(field)
	} else {
		hli.ir.LogError(fmt.Errorf("can't add field: %s", err))
	}
	if field, err := message.NewField("ContentType", req.Header.Get("Content-Type"), ""); err == nil {
		pack.Message.AddField(field)
	} else {
		hli.ir.LogError(fmt.Errorf("can't add field: %s", err))
	}

	for key, values := range req.URL.Query() {
		for i := range values {
			value := values[i]
			if field, err := message.NewField(key, value, ""); err == nil {
				pack.Message.AddField(field)
			} else {
				hli.ir.LogError(fmt.Errorf("can't add field: %s", err))
			}
		}
	}

	if hli.dRunner == nil {
		hli.ir.Inject(pack)
	} else {
		hli.dRunner.InChan() <- pack
	}
}

func getStatsdPack(typeStr string, payload string) (pack *pipeline.PipelinePack) {
	recycleChan := make(chan *pipeline.PipelinePack, 1)
	pack = pipeline.NewPipelinePack(recycleChan)
	pack.Message.SetType(typeStr)
	pack.Message.SetLogger("thenamespace")
	fName, _ := message.NewField("name", "myname", "")
	fRate, _ := message.NewField("rate", .30, "")
	pack.Message.AddField(fName)
	pack.Message.AddField(fRate)
	pack.Message.SetPayload(payload)
	pack.Decoded = true
	return pack
}

func GeoIpDecoderSpec(c gs.Context) {
	t := &ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	pConfig := NewPipelineConfig(nil)
	pConfig.Globals.ShareDir = "/foo/bar/baz"

	c.Specify("A GeoIpDecoder", func() {
		decoder := new(GeoIpDecoder)
		decoder.SetPipelineConfig(pConfig)
		rec := new(geoip.GeoIPRecord)
		conf := decoder.ConfigStruct().(*GeoIpDecoderConfig)

		c.Expect(conf.DatabaseFile, gs.Equals, "/foo/bar/baz/GeoLiteCity.dat")

		supply := make(chan *PipelinePack, 1)
		pack := NewPipelinePack(supply)

		nf, _ := message.NewField("remote_host", "74.125.142.147", "")
		pack.Message.AddField(nf)

		decoder.SourceIpField = "remote_host"
		conf.SourceIpField = "remote_host"
		decoder.Init(conf)

		rec.CountryCode = "US"
		rec.CountryCode3 = "USA"
		rec.CountryName = "United States"
		rec.Region = "CA"
		rec.City = "Mountain View"
		rec.PostalCode = "94043"
		rec.Latitude = 37.4192
		rec.Longitude = -122.0574
		rec.AreaCode = 650
		rec.CharSet = 1
		rec.ContinentCode = "NA"

		c.Specify("Test GeoIpDecoder Output", func() {
			buf := decoder.GeoBuff(rec)
			nf, _ = message.NewField("geoip", buf.Bytes(), "")
			pack.Message.AddField(nf)

			b, ok := pack.Message.GetFieldValue("geoip")
			c.Expect(ok, gs.IsTrue)

			c.Expect(string(b.([]byte)), gs.Equals, `{"latitude":37.4192008972168,"longitude":-122.0574035644531,"location":[-122.0574035644531,37.4192008972168],"coordinates":["-122.0574035644531","37.4192008972168"],"countrycode":"US","countrycode3":"USA","countryname":"United States","region":"CA","city":"Mountain View","postalcode":"94043","areacode":650,"charset":1,"continentcode":"NA"}`)
		})

	})
}

func getTestMessage() *message.Message {
	hostname, _ := os.Hostname()
	field, _ := message.NewField("foo", "bar", "")
	msg := &message.Message{}
	msg.SetType("TEST")
	msg.SetTimestamp(5123456789)
	msg.SetPid(9283)
	msg.SetUuid(uuid.NewRandom())
	msg.SetLogger("GoSpec")
	msg.SetSeverity(int32(6))
	msg.SetEnvVersion("0.8")
	msg.SetPid(int32(os.Getpid()))
	msg.SetHostname(hostname)
	msg.AddField(field)

	var emptyByte []byte
	data := []byte("data")
	field1, _ := message.NewField("bytes", data, "")
	field2, _ := message.NewField("int", int64(999), "")
	field2.AddValue(int64(1024))
	field3, _ := message.NewField("double", float64(99.9), "")
	field4, _ := message.NewField("bool", true, "")
	field5, _ := message.NewField("foo", "alternate", "")
	field6, _ := message.NewField("false", false, "")
	field7, _ := message.NewField("empty_bytes", emptyByte, "")
	msg.AddField(field1)
	msg.AddField(field2)
	msg.AddField(field3)
	msg.AddField(field4)
	msg.AddField(field5)
	msg.AddField(field6)
	msg.AddField(field7)
	return msg
}

func SentryOutputSpec(c gs.Context) {

	pack := getSentryPack()
	output := new(SentryOutput)
	output.Init(output.ConfigStruct())
	sentryMsg := &SentryMsg{
		dataPacket: make([]byte, 0, output.config.MaxSentryBytes),
	}
	var err error

	c.Specify("verify data_packet bytes", func() {
		err = output.prepSentryMsg(pack, sentryMsg)
		c.Expect(err, gs.Equals, nil)

		actual := string(sentryMsg.dataPacket)
		ts := int64(EPOCH_TS * 1e9)
		t := time.Unix(ts/1e9, ts%1e9)
		expected := fmt.Sprintf("Sentry sentry_timestamp=%s, "+
			"sentry_client=raven-go/1.0, sentry_version=2.0, "+
			"sentry_key=username\n\n%s", t.Format(time.RFC3339Nano), PAYLOAD)

		c.Expect(actual, gs.Equals, expected)
	})

	c.Specify("missing or invalid epoch_timestamp doesn't kill process", func() {
		f := pack.Message.FindFirstField("epoch_timestamp")
		*f.Name = "other"
		err = output.prepSentryMsg(pack, sentryMsg)
		c.Expect(err.Error(), gs.Equals, "no `epoch_timestamp` field")

		f, _ = message.NewField("epoch_timestamp", "foo", "")
		pack.Message.AddField(f)
		err = output.prepSentryMsg(pack, sentryMsg)
		c.Expect(err.Error(), gs.Equals, "`epoch_timestamp` isn't a float64")
	})

	c.Specify("missing or invalid dsn doesn't kill process", func() {
		f := pack.Message.FindFirstField("dsn")
		*f.Name = "other"
		err = output.prepSentryMsg(pack, sentryMsg)
		c.Expect(err.Error(), gs.Equals, "no `dsn` field")

		f, _ = message.NewField("dsn", 42, "")
		pack.Message.AddField(f)
		err = output.prepSentryMsg(pack, sentryMsg)
		c.Expect(err.Error(), gs.Equals, "`dsn` isn't a string")
	})

}

func (f *StreamAggregatorFilter) committer(fr FilterRunner, h PluginHelper, wg *sync.WaitGroup) {
	initBatch := make([]byte, 0, 10000)
	f.backChan <- initBatch
	var (
		tag      string
		outBatch []byte
	)
	tag = f.StreamAggregatorTag

	for outBatch = range f.batchChan {
		pack, e := h.PipelinePack(f.msgLoopCount)
		if e != nil {
			fr.LogError(e)
			break
		}

		tagField, _ := message.NewField("StreamAggregatorTag", tag, "")
		pack.Message.AddField(tagField)
		pack.Message.SetUuid(uuid.NewRandom())
		if f.OutputType == "json" {
			jsonStr := strings.TrimRight(string(outBatch), ",")
			pack.Message.SetPayload("[" + jsonStr + "]")
		} else {
			pack.Message.SetPayload(string(outBatch))
		}
		fr.Inject(pack)

		outBatch = outBatch[:0]
		f.backChan <- outBatch
	}
	wg.Done()
}

func (f *ZlibFilter) committer(fr FilterRunner, h PluginHelper, wg *sync.WaitGroup) {
	initBatch := make([]byte, 0, 10000)
	f.backChan <- initBatch
	var (
		tag string
		//ok bool
		outBatch []byte
	)
	tag = f.ZlibTag

	for outBatch = range f.batchChan {
		pack, e := h.PipelinePack(f.msgLoopCount)
		if e != nil {
			fr.LogError(e)
			break
		}
		var b bytes.Buffer
		w := zlib.NewWriter(&b)
		w.Write(outBatch)
		w.Close()

		tagField, _ := message.NewField("ZlibTag", tag, "")
		pack.Message.AddField(tagField)
		pack.Message.SetUuid(uuid.NewRandom())
		pack.Message.SetPayload(b.String())
		fr.Inject(pack)

		outBatch = outBatch[:0]
		f.backChan <- outBatch
	}
	wg.Done()
}

func (ld *GeoIpDecoder) Decode(pack *PipelinePack) (packs []*PipelinePack, err error) {
	var buf bytes.Buffer
	var ipAddr, _ = pack.Message.GetFieldValue(ld.SourceIpField)

	ip, ok := ipAddr.(string)

	if !ok {
		// IP field was not a string. Field could just be blank. Return without error.
		packs = []*PipelinePack{pack}
		return
	}

	if ld.gi != nil {
		rec := ld.gi.GetRecord(ip)
		if rec != nil {
			buf = ld.GeoBuff(rec)
		} else {
			// IP address did not return a valid GeoIp record but that's ok sometimes(private ip?). Return without error.
			packs = []*PipelinePack{pack}
			return
		}
	}

	if buf.Len() > 0 {
		var nf *message.Field
		nf, err = message.NewField(ld.TargetField, buf.Bytes(), "")
		pack.Message.AddField(nf)
	}

	packs = []*PipelinePack{pack}

	return
}

// Convenience function for creating and setting a string field called "name"
// on a message object.
func newStringField(msg *message.Message, name string, val string) {
	if f, err := message.NewField(name, val, ""); err == nil {
		msg.AddField(f)
	} else {
		fmt.Println("Report error adding string field: ", err)
	}
}

// Applies this message template's values to the provided message object,
// interpolating the provided substitutions into the values in the process.
func (mt MessageTemplate) PopulateMessage(msg *message.Message, subs map[string]string) error {
	var val string
	for field, rawVal := range mt {
		val = InterpolateString(rawVal, subs)
		switch field {
		case "Logger":
			msg.SetLogger(val)
		case "Type":
			msg.SetType(val)
		case "Payload":
			msg.SetPayload(val)
		case "Hostname":
			msg.SetHostname(val)
		case "Pid":
			pid, err := strconv.ParseInt(val, 10, 32)
			if err != nil {
				return err
			}
			msg.SetPid(int32(pid))
		case "Uuid":
			msg.SetUuid([]byte(val))
		default:
			fi := strings.SplitN(field, "|", 2)
			if len(fi) < 2 {
				fi = append(fi, "")
			}
			f, err := message.NewField(fi[0], val, fi[1])
			msg.AddField(f)
			if err != nil {
				return err
			}
		}
	}
	return nil
}