Golang pipeline.NewPipelineConfig函數代碼示例

func TestLoadDir(t *testing.T) {
	origGlobals := pipeline.Globals
	origAvailablePlugins := make(map[string]func() interface{})
	for k, v := range pipeline.AvailablePlugins {
		origAvailablePlugins[k] = v
	}

	defer func() {
		pipeline.Globals = origGlobals
		pipeline.AvailablePlugins = origAvailablePlugins
	}()

	pipeConfig := pipeline.NewPipelineConfig(nil)
	confDirPath := "../../plugins/testsupport/config_dir"
	err := loadFullConfig(pipeConfig, &confDirPath)
	if err != nil {
		t.Fatal(err)
	}

	// verify the inputs sections load properly with a custom name
	udp, ok := pipeConfig.InputRunners["UdpInput"]
	if !ok {
		t.Fatal("No UdpInput configured.")
	}
	defer udp.Input().Stop()

	// and the decoders sections load
	_, ok = pipeConfig.DecoderWrappers["ProtobufDecoder"]
	if !ok {
		t.Fatal("No ProtobufDecoder configured.")
	}

	// and the outputs sections load
	_, ok = pipeConfig.OutputRunners["LogOutput"]
	if !ok {
		t.Fatal("No LogOutput configured")
	}

	// and the filters sections load
	_, ok = pipeConfig.FilterRunners["sample"]
	if !ok {
		t.Fatal("No `sample` filter configured.")
	}

	// and the encoders sections load
	encoder, ok := pipeConfig.Encoder("PayloadEncoder", "foo")
	if !ok {
		t.Fatal("No PayloadEncoder configured.")
	}
	_, ok = encoder.(*plugins.PayloadEncoder)
	if !ok {
		t.Fatal("PayloadEncoder isn't a PayloadEncoder")
	}

	// and that the non "*.toml" file did *not* load
	_, ok = pipeConfig.FilterRunners["not_loaded"]
	if ok {
		t.Fatal("`not_loaded` filter *was* loaded, shouldn't have been!")
	}
}

func main() {
	configFile := flag.String("config", "agent.conf", "Agent Config file")
	maxprocs := flag.Int("maxprocs", 1, "Go runtime MAXPROCS value")
	poolSize := flag.Int("poolsize", 1000, "Pipeline pool size")
	pprofName := flag.String("pprof", "", "Go profiler output file")
	version := flag.Bool("version", false, "Output version and exit")
	flag.Parse()

	if *version {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	runtime.GOMAXPROCS(*maxprocs)

	if *pprofName != "" {
		profFile, err := os.Create(*pprofName)
		if err != nil {
			log.Fatalln(err)
		}
		pprof.StartCPUProfile(profFile)
		defer pprof.StopCPUProfile()
	}

	// Set up and load the pipeline configuration and start the daemon.
	pipeconf := pipeline.NewPipelineConfig(*poolSize)
	err := pipeconf.LoadFromConfigFile(*configFile)
	if err != nil {
		log.Fatal("Error reading config: ", err)
	}
	pipeline.Run(pipeconf)
}

func main() {
	configFile := flag.String("config", "/etc/hekad.toml", "Config file")
	maxprocs := flag.Int("maxprocs", 1, "Go runtime MAXPROCS value")
	poolSize := flag.Int("poolsize", 100, "Pipeline pool size")
	decoderPoolSize := flag.Int("decoder_poolsize", 4, "Decoder pool size")
	chanSize := flag.Int("plugin_chansize", 50, "Plugin input channel buffer size")
	cpuProfName := flag.String("cpuprof", "", "Go CPU profiler output file")
	memProfName := flag.String("memprof", "", "Go memory profiler output file")
	version := flag.Bool("version", false, "Output version and exit")
	maxMsgLoops := flag.Uint("max_message_loops", 4, "Maximum number of times a message can pass thru the system")
	flag.Parse()

	if *version {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	runtime.GOMAXPROCS(*maxprocs)

	if *cpuProfName != "" {
		profFile, err := os.Create(*cpuProfName)
		if err != nil {
			log.Fatalln(err)
		}
		pprof.StartCPUProfile(profFile)
		defer pprof.StopCPUProfile()
	}

	if *memProfName != "" {
		defer func() {
			profFile, err := os.Create(*memProfName)
			if err != nil {
				log.Fatalln(err)
			}
			pprof.WriteHeapProfile(profFile)
			profFile.Close()
		}()
	}

	// Set up and load the pipeline configuration and start the daemon.
	globals := pipeline.DefaultGlobals()
	globals.PoolSize = *poolSize
	globals.DecoderPoolSize = *decoderPoolSize
	globals.PluginChanSize = *chanSize
	globals.MaxMsgLoops = *maxMsgLoops
	if globals.MaxMsgLoops == 0 {
		globals.MaxMsgLoops = 1
	}
	pipeconf := pipeline.NewPipelineConfig(globals)
	err := pipeconf.LoadFromConfigFile(*configFile)
	if err != nil {
		log.Fatal("Error reading config: ", err)
	}
	pipeline.Run(pipeconf)
}

func TestCustomHostname(t *testing.T) {
	expected := "my.example.com"
	configPath := "../../pipeline/testsupport/sample-hostname.toml"
	config, err := LoadHekadConfig(configPath)
	if err != nil {
		t.Fatal(err)
	}
	if config.Hostname != expected {
		t.Fatalf("HekadConfig.Hostname expected: '%s', Got: %s", expected, config.Hostname)
	}
	globals, _, _ := setGlobalConfigs(config)
	if globals.Hostname != expected {
		t.Fatalf("globals.Hostname expected: '%s', Got: %s", expected, globals.Hostname)
	}
	pConfig := pipeline.NewPipelineConfig(globals)
	if pConfig.Hostname() != expected {
		t.Fatalf("PipelineConfig.Hostname expected: '%s', Got: %s", expected, pConfig.Hostname())
	}
	err = loadFullConfig(pConfig, &configPath)
	if err != nil {
		t.Fatalf("Error loading full config: %s", err.Error())
	}
}

func main() {
	configFile := flag.String("config", "/etc/hekad.toml", "Config file")
	version := flag.Bool("version", false, "Output version and exit")
	flag.Parse()

	if flag.NFlag() == 0 {
		flag.PrintDefaults()
		os.Exit(0)
	}

	if *version {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	config, err := LoadHekadConfig(*configFile)
	if err != nil {
		log.Fatal("Error reading config: ", err)
	}

	maxprocs := config.Maxprocs
	poolSize := config.PoolSize
	decoderPoolSize := config.DecoderPoolSize
	chanSize := config.ChanSize
	cpuProfName := config.CpuProfName
	memProfName := config.MemProfName
	maxMsgLoops := config.MaxMsgLoops
	maxMsgProcessInject := config.MaxMsgProcessInject
	maxMsgTimerInject := config.MaxMsgTimerInject

	runtime.GOMAXPROCS(maxprocs)

	if cpuProfName != "" {
		profFile, err := os.Create(cpuProfName)
		if err != nil {
			log.Fatalln(err)
		}
		profFile.Close()
		pprof.StartCPUProfile(profFile)
		defer pprof.StopCPUProfile()
	}

	if memProfName != "" {
		defer func() {
			profFile, err := os.Create(memProfName)
			if err != nil {
				log.Fatalln(err)
			}
			pprof.WriteHeapProfile(profFile)
			profFile.Close()
		}()
	}

	// Set up and load the pipeline configuration and start the daemon.
	globals := pipeline.DefaultGlobals()
	globals.PoolSize = poolSize
	globals.DecoderPoolSize = decoderPoolSize
	globals.PluginChanSize = chanSize
	globals.MaxMsgLoops = maxMsgLoops
	if globals.MaxMsgLoops == 0 {
		globals.MaxMsgLoops = 1
	}
	globals.MaxMsgProcessInject = maxMsgProcessInject
	globals.MaxMsgTimerInject = maxMsgTimerInject
	pipeconf := pipeline.NewPipelineConfig(globals)

	err = pipeconf.LoadFromConfigFile(*configFile)
	if err != nil {
		log.Fatal("Error reading config: ", err)
	}
	pipeline.Run(pipeconf)
}

func DashboardOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	pConfig := pipeline.NewPipelineConfig(nil)

	dashboardOutput := new(DashboardOutput)
	dashboardOutput.pConfig = pConfig

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	oth.MockHelper = pipelinemock.NewMockPluginHelper(ctrl)
	oth.MockOutputRunner = pipelinemock.NewMockOutputRunner(ctrl)

	errChan := make(chan error, 1)

	startOutput := func() {
		go func() {
			errChan <- dashboardOutput.Run(oth.MockOutputRunner, oth.MockHelper)
		}()
	}

	if runtime.GOOS != "windows" {
		c.Specify("A DashboardOutput", func() {

			tmpdir, err := ioutil.TempDir("", "dashboard_output_test")
			c.Assume(err, gs.IsNil)
			config := dashboardOutput.ConfigStruct().(*DashboardOutputConfig)
			config.WorkingDirectory = tmpdir

			c.Specify("Init halts if basedirectory is not writable", func() {
				err := os.MkdirAll(tmpdir, 0400)
				c.Assume(err, gs.IsNil)
				defer os.RemoveAll(tmpdir)
				err = dashboardOutput.Init(config)
				c.Assume(err, gs.Not(gs.IsNil))
			})

			c.Specify("that is running", func() {
				startedChan := make(chan bool, 1)
				defer close(startedChan)
				ts := httptest.NewUnstartedServer(nil)

				dashboardOutput.starterFunc = func(hli *DashboardOutput) error {
					ts.Start()
					startedChan <- true
					return nil
				}

				ticker := make(chan time.Time)
				inChan := make(chan *pipeline.PipelinePack, 1)
				recycleChan := make(chan *pipeline.PipelinePack, 1)
				pack := pipeline.NewPipelinePack(recycleChan)
				pack.Message = pipeline_ts.GetTestMessage()

				oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
				oth.MockOutputRunner.EXPECT().Ticker().Return(ticker)

				err := os.MkdirAll(tmpdir, 0700)
				c.Assume(err, gs.IsNil)
				defer os.RemoveAll(tmpdir)

				dashboardOutput.handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					// Noop
				})

				c.Specify("sets custom http headers", func() {
					config.Headers = http.Header{
						"One":  []string{"two", "three"},
						"Four": []string{"five", "six", "seven"},
					}
					err = dashboardOutput.Init(config)
					c.Assume(err, gs.IsNil)
					ts.Config = dashboardOutput.server

					startOutput()

					inChan <- pack
					<-startedChan
					resp, err := http.Get(ts.URL)
					c.Assume(err, gs.IsNil)
					resp.Body.Close()
					c.Assume(resp.StatusCode, gs.Equals, 200)

					// Verify headers are there
					eq := reflect.DeepEqual(resp.Header["One"], config.Headers["One"])
					c.Expect(eq, gs.IsTrue)
					eq = reflect.DeepEqual(resp.Header["Four"], config.Headers["Four"])
					c.Expect(eq, gs.IsTrue)
				})

				close(inChan)
				c.Expect(<-errChan, gs.IsNil)

				ts.Close()
			})
		})
	}
}

func EncoderSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	// NewPipelineConfig sets up Globals which is needed for the
	// pipeline.Prepend*Dir functions to not die during plugin Init().
	_ = pipeline.NewPipelineConfig(nil)

	c.Specify("A SandboxEncoder", func() {

		encoder := new(SandboxEncoder)
		conf := encoder.ConfigStruct().(*SandboxEncoderConfig)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		pack.Message.SetPayload("original")
		pack.Message.SetType("my_type")
		pack.Message.SetPid(12345)
		pack.Message.SetSeverity(4)
		pack.Message.SetHostname("hostname")
		pack.Message.SetTimestamp(54321)
		pack.Message.SetUuid(uuid.NewRandom())
		var (
			result []byte
			err    error
		)

		c.Specify("emits JSON correctly", func() {
			conf.ScriptFilename = "../lua/testsupport/encoder_json.lua"
			err = encoder.Init(conf)
			c.Expect(err, gs.IsNil)

			result, err = encoder.Encode(pack)
			c.Expect(err, gs.IsNil)
			msg := new(message.Message)
			err = json.Unmarshal(result, msg)
			c.Expect(err, gs.IsNil)
			c.Expect(msg.GetTimestamp(), gs.Equals, int64(54321))
			c.Expect(msg.GetPid(), gs.Equals, int32(12345))
			c.Expect(msg.GetSeverity(), gs.Equals, int32(4))
			c.Expect(msg.GetHostname(), gs.Equals, "hostname")
			c.Expect(msg.GetPayload(), gs.Equals, "original")
			c.Expect(msg.GetType(), gs.Equals, "my_type")
		})

		c.Specify("emits text correctly", func() {
			conf.ScriptFilename = "../lua/testsupport/encoder_text.lua"
			err = encoder.Init(conf)
			c.Expect(err, gs.IsNil)

			result, err = encoder.Encode(pack)
			c.Expect(err, gs.IsNil)
			c.Expect(string(result), gs.Equals, "Prefixed original")
		})

		c.Specify("emits protobuf correctly", func() {

			c.Specify("when inject_message is used", func() {
				conf.ScriptFilename = "../lua/testsupport/encoder_protobuf.lua"
				err = encoder.Init(conf)
				c.Expect(err, gs.IsNil)

				result, err = encoder.Encode(pack)
				c.Expect(err, gs.IsNil)

				msg := new(message.Message)
				err = proto.Unmarshal(result, msg)
				c.Expect(err, gs.IsNil)
				c.Expect(msg.GetTimestamp(), gs.Equals, int64(54321))
				c.Expect(msg.GetPid(), gs.Equals, int32(12345))
				c.Expect(msg.GetSeverity(), gs.Equals, int32(4))
				c.Expect(msg.GetHostname(), gs.Equals, "hostname")
				c.Expect(msg.GetPayload(), gs.Equals, "mutated")
				c.Expect(msg.GetType(), gs.Equals, "after")
			})

			c.Specify("when `write_message` is used", func() {
				conf.ScriptFilename = "../lua/testsupport/encoder_writemessage.lua"
				err = encoder.Init(conf)
				c.Expect(err, gs.IsNil)

				result, err = encoder.Encode(pack)
				c.Expect(err, gs.IsNil)

				msg := new(message.Message)
				err = proto.Unmarshal(result, msg)
				c.Expect(err, gs.IsNil)
				c.Expect(msg.GetPayload(), gs.Equals, "mutated payload")
				c.Expect(pack.Message.GetPayload(), gs.Equals, "original")
			})
		})

	})
}

func FilterSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	fth := NewFilterTestHelper(ctrl)
	inChan := make(chan *pipeline.PipelinePack, 1)
	pConfig := pipeline.NewPipelineConfig(nil)

	c.Specify("A SandboxFilter", func() {
		sbFilter := new(SandboxFilter)
		sbFilter.SetPipelineConfig(pConfig)
		config := sbFilter.ConfigStruct().(*sandbox.SandboxConfig)
		config.MemoryLimit = 64000
		config.InstructionLimit = 1000
		config.OutputLimit = 1024

		msg := getTestMessage()
		pack := pipeline.NewPipelinePack(pConfig.InjectRecycleChan())
		pack.Message = msg

		c.Specify("Uninitialized", func() {
			err := sbFilter.ReportMsg(msg)
			c.Expect(err, gs.IsNil)
		})

		c.Specify("Over inject messages from ProcessMessage", func() {
			var timer <-chan time.Time
			fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
			fth.MockFilterRunner.EXPECT().Name().Return("processinject").Times(2)
			fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(2)
			fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil).Times(2)
			fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)

			config.ScriptFilename = "../lua/testsupport/processinject.lua"
			config.ModuleDirectory = "../lua/modules"
			err := sbFilter.Init(config)
			c.Assume(err, gs.IsNil)
			inChan <- pack
			close(inChan)
			err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			var termErr pipeline.TerminatedError
			if runtime.GOOS == "windows" {
				termErr = pipeline.TerminatedError("process_message() ..\\lua\\testsupport\\processinject.lua:8: inject_payload() exceeded InjectMessage count")
			} else {
				termErr = pipeline.TerminatedError("process_message() ../lua/testsupport/processinject.lua:8: inject_payload() exceeded InjectMessage count")
			}
			c.Expect(err.Error(), gs.Equals, termErr.Error())
		})

		c.Specify("Over inject messages from TimerEvent", func() {
			var timer <-chan time.Time
			timer = time.Tick(time.Duration(1) * time.Millisecond)
			fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
			fth.MockFilterRunner.EXPECT().Name().Return("timerinject").Times(11)
			fth.MockFilterRunner.EXPECT().Inject(pack).Return(true).Times(11)
			fth.MockHelper.EXPECT().PipelinePack(uint(0)).Return(pack, nil).Times(11)
			fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)

			config.ScriptFilename = "../lua/testsupport/timerinject.lua"
			config.ModuleDirectory = "../lua/modules"
			err := sbFilter.Init(config)
			c.Assume(err, gs.IsNil)
			go func() {
				time.Sleep(time.Duration(250) * time.Millisecond)
				close(inChan)
			}()
			err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)

			var termErr pipeline.TerminatedError
			if runtime.GOOS == "windows" {
				termErr = pipeline.TerminatedError("timer_event() ..\\lua\\testsupport\\timerinject.lua:13: inject_payload() exceeded InjectMessage count")
			} else {
				termErr = pipeline.TerminatedError("timer_event() ../lua/testsupport/timerinject.lua:13: inject_payload() exceeded InjectMessage count")
			}
			c.Expect(err.Error(), gs.Equals, termErr.Error())
		})

		c.Specify("Preserves data", func() {
			var timer <-chan time.Time
			fth.MockFilterRunner.EXPECT().Ticker().Return(timer)
			fth.MockFilterRunner.EXPECT().InChan().Return(inChan)
			fth.MockFilterRunner.EXPECT().UsesBuffering().Return(true)
			fth.MockHelper.EXPECT().PipelineConfig().Return(pConfig)

			config.ScriptFilename = "../lua/testsupport/serialize.lua"
			config.ModuleDirectory = "../lua/modules"
			config.PreserveData = true
			sbFilter.SetName("serialize")
			err := sbFilter.Init(config)
			c.Assume(err, gs.IsNil)
			close(inChan)
			err = sbFilter.Run(fth.MockFilterRunner, fth.MockHelper)
			c.Expect(err, gs.IsNil)
			_, err = os.Stat("sandbox_preservation/serialize.data")
			c.Expect(err, gs.IsNil)
//.........這裏部分代碼省略.........

func DecoderSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	// NewPipelineConfig sets up Globals which is needed for the
	// pipeline.Prepend*Dir functions to not die during plugin Init().
	_ = pipeline.NewPipelineConfig(nil)

	c.Specify("A SandboxDecoder", func() {

		decoder := new(SandboxDecoder)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)
		conf.ScriptType = "lua"

		c.Specify("that uses lpeg and inject_message", func() {
			dRunner.EXPECT().Name().Return("serialize")
			conf.ScriptFilename = "../lua/testsupport/decoder.lua"
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			c.Specify("decodes simple messages", func() {
				data := "1376389920 debug id=2321 url=example.com item=1"
				decoder.SetDecoderRunner(dRunner)
				pack.Message.SetPayload(data)
				_, err = decoder.Decode(pack)
				c.Assume(err, gs.IsNil)

				c.Expect(pack.Message.GetTimestamp(),
					gs.Equals,
					int64(1376389920000000000))

				c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

				var ok bool
				var value interface{}
				value, ok = pack.Message.GetFieldValue("id")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "2321")

				value, ok = pack.Message.GetFieldValue("url")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "example.com")

				value, ok = pack.Message.GetFieldValue("item")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "1")
				decoder.Shutdown()
			})

			c.Specify("decodes an invalid messages", func() {
				data := "1376389920 bogus id=2321 url=example.com item=1"
				decoder.SetDecoderRunner(dRunner)
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(len(packs), gs.Equals, 0)
				c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
				c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
				decoder.Shutdown()
			})

			c.Specify("Preserves data", func() {
				conf.ScriptFilename = "../lua/testsupport/serialize.lua"
				conf.PreserveData = true
				err := decoder.Init(conf)
				c.Assume(err, gs.IsNil)
				decoder.SetDecoderRunner(dRunner)
				decoder.Shutdown()
				_, err = os.Stat("sandbox_preservation/serialize.data")
				c.Expect(err, gs.IsNil)
				err = os.Remove("sandbox_preservation/serialize.data")
				c.Expect(err, gs.IsNil)
			})
		})

		c.Specify("that only uses write_message", func() {
			conf.ScriptFilename = "../lua/testsupport/write_message_decoder.lua"
			dRunner.EXPECT().Name().Return("write_message")
			err := decoder.Init(conf)
			decoder.SetDecoderRunner(dRunner)
			c.Assume(err, gs.IsNil)

			c.Specify("adds a string field to the message", func() {
				data := "string field scribble"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				value, ok := pack.Message.GetFieldValue("scribble")
				c.Expect(ok, gs.IsTrue)
				c.Expect(value.(string), gs.Equals, "foo")
			})

			c.Specify("adds a numeric field to the message", func() {
				data := "num field scribble"
				pack.Message.SetPayload(data)
//.........這裏部分代碼省略.........

func HttpInputSpec(c gs.Context) {
	t := &ts.SimpleT{}
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := pipeline.NewPipelineConfig(nil)

	jsonPost := `{"uuid": "xxBI3zyeXU+spG8Uiveumw==", "timestamp": 1372966886023588, "hostname": "Victors-MacBook-Air.local", "pid": 40183, "fields": [{"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_priority", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_ident", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_facility", "value_string": [""]}, {"representation": "", "value_type": "STRING", "name": "cef_meta.syslog_options", "value_string": [""]}], "logger": "", "env_version": "0.8", "type": "cef", "payload": "Jul 04 15:41:26 Victors-MacBook-Air.local CEF:0|mozilla|weave|3|xx\\\\|x|xx\\\\|x|5|cs1Label=requestClientApplication cs1=MySuperBrowser requestMethod=GET request=/ src=127.0.0.1 dest=127.0.0.1 suser=none", "severity": 6}'`

	c.Specify("A HttpInput", func() {

		httpInput := pipeline.HttpInput{}

		c.Specify("honors time ticker to flush", func() {
			ith := new(pipeline.InputTestHelper)
			ith.MockHelper = NewMockPluginHelper(ctrl)
			ith.MockInputRunner = NewMockInputRunner(ctrl)
			startInput := func() {
				go func() {
					err := httpInput.Run(ith.MockInputRunner, ith.MockHelper)
					c.Expect(err, gs.IsNil)
				}()
			}

			ith.Pack = NewPipelinePack(pConfig.inputRecycleChan)
			ith.PackSupply = make(chan *PipelinePack, 1)
			ith.PackSupply <- ith.Pack

			ith.Decoders = make([]DecoderRunner, int(message.Header_JSON+1))
			ith.Decoders[message.Header_JSON] = NewMockDecoderRunner(ctrl)
			ith.MockDecoderSet = NewMockDecoderSet(ctrl)

			// Spin up a http server
			server, err := ts.NewOneHttpServer(jsonPost, "localhost", 9876)
			c.Expect(err, gs.IsNil)
			go server.Start("/")
			time.Sleep(10 * time.Millisecond)

			config := httpInput.ConfigStruct().(*HttpInputConfig)
			config.DecoderName = "JsonDecoder"
			config.Url = "http://localhost:9876/"
			tickChan := make(chan time.Time)

			ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)
			ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)

			ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
			ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
			ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)

			mockDecoderRunner := ith.Decoders[message.Header_JSON].(*MockDecoderRunner)

			// Stub out the DecoderRunner input channel so that we can
			// inspect bytes later on
			dRunnerInChan := make(chan *PipelinePack, 1)
			mockDecoderRunner.EXPECT().InChan().Return(dRunnerInChan)

			dset := ith.MockDecoderSet.EXPECT().ByName("JsonDecoder")
			dset.Return(ith.Decoders[message.Header_JSON], true)

			err = httpInput.Init(config)
			c.Assume(err, gs.IsNil)
			startInput()

			tickChan <- time.Now()

			// We need for the pipeline to finish up
			time.Sleep(50 * time.Millisecond)
		})

		c.Specify("short circuits packs into the router", func() {
			ith := new(InputTestHelper)
			ith.MockHelper = NewMockPluginHelper(ctrl)
			ith.MockInputRunner = NewMockInputRunner(ctrl)
			startInput := func() {
				go func() {
					err := httpInput.Run(ith.MockInputRunner, ith.MockHelper)
					c.Expect(err, gs.IsNil)
				}()
			}
			ith.Pack = NewPipelinePack(pConfig.inputRecycleChan)
			ith.PackSupply = make(chan *PipelinePack, 1)
			ith.PackSupply <- ith.Pack

			ith.Decoders = make([]DecoderRunner, int(message.Header_JSON+1))
			ith.Decoders[message.Header_JSON] = NewMockDecoderRunner(ctrl)
			ith.MockDecoderSet = NewMockDecoderSet(ctrl)
			config := httpInput.ConfigStruct().(*HttpInputConfig)
			config.Url = "http://localhost:9876/"
			tickChan := make(chan time.Time)

			ith.MockInputRunner.EXPECT().LogMessage(gomock.Any()).Times(2)
			ith.MockHelper.EXPECT().DecoderSet().Return(ith.MockDecoderSet)

			ith.MockHelper.EXPECT().PipelineConfig().Return(pConfig)
			ith.MockInputRunner.EXPECT().InChan().Return(ith.PackSupply)
			ith.MockInputRunner.EXPECT().Ticker().Return(tickChan)

			err := httpInput.Init(config)
			c.Assume(err, gs.IsNil)
//.........這裏部分代碼省略.........

func NsqOutputSpec(c gs.Context) {
	t := new(pipeline_ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := pipeline.NewPipelineConfig(nil)
	var wg sync.WaitGroup

	errChan := make(chan error, 1)
	defer close(errChan)

	c.Specify("A nsq output", func() {
		output := new(NsqOutput)
		oth := plugins_ts.NewOutputTestHelper(ctrl)
		config := output.ConfigStruct().(*NsqOutputConfig)
		config.Topic = "test"

		startOutput := func() {
			wg.Add(1)
			go func() {
				errChan <- output.Run(oth.MockOutputRunner, oth.MockHelper)
				wg.Done()
			}()
		}

		var mockProducer *MockProducer
		output.newProducer = func(addr string, config *nsq.Config) (Producer, error) {
			mockProducer, _ = NewMockProducer(addr, config)
			return mockProducer, nil
		}

		msg := pipeline_ts.GetTestMessage()
		pack := pipeline.NewPipelinePack(pConfig.InputRecycleChan())
		pack.Message = msg

		c.Specify("requires at least one address", func() {
			err := output.Init(config)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		config.Addresses = []string{"test.com:4150"}

		c.Specify("requires an encoder", func() {
			err := output.Init(config)
			c.Assume(err, gs.IsNil)

			oth.MockOutputRunner.EXPECT().Encoder().Return(nil)
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		c.Specify("that is started", func() {
			encoder := new(plugins.PayloadEncoder)
			encoder.Init(&plugins.PayloadEncoderConfig{PrefixTs: false})
			oth.MockOutputRunner.EXPECT().Encoder().Return(encoder)
			oth.MockOutputRunner.EXPECT().Encode(pack).Return(encoder.Encode(pack))

			inChan := make(chan *pipeline.PipelinePack, 1)
			oth.MockOutputRunner.EXPECT().InChan().Return(inChan)

			c.Specify("publishes a message with the configured topic", func() {
				err := output.Init(config)
				c.Assume(err, gs.IsNil)

				startOutput()

				inChan <- pack
				transaction := <-mockProducer.respChan
				c.Expect(transaction.Error, gs.IsNil)

				close(inChan)
				wg.Wait()
				c.Expect(<-errChan, gs.IsNil)
			})
		})

	})
}

func OutputSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	oth := plugins_ts.NewOutputTestHelper(ctrl)
	pConfig := pipeline.NewPipelineConfig(nil)
	inChan := make(chan *pipeline.PipelinePack, 1)
	timer := make(chan time.Time, 1)

	c.Specify("A SandboxOutput", func() {
		output := new(SandboxOutput)
		output.SetPipelineConfig(pConfig)
		conf := output.ConfigStruct().(*sandbox.SandboxConfig)
		conf.ScriptFilename = "../lua/testsupport/output.lua"
		conf.ModuleDirectory = "../lua/modules;../lua/testsupport/modules"
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		data := "1376389920 debug id=2321 url=example.com item=1"
		pack.Message.SetPayload(data)

		oth.MockOutputRunner.EXPECT().InChan().Return(inChan)
		oth.MockOutputRunner.EXPECT().UpdateCursor("").AnyTimes()
		oth.MockOutputRunner.EXPECT().Ticker().Return(timer)

		c.Specify("writes a payload to file", func() {
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			inChan <- pack
			close(inChan)
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Assume(err, gs.IsNil)

			tmpFile, err := os.Open("output.lua.txt")
			defer tmpFile.Close()
			c.Assume(err, gs.IsNil)
			contents, err := ioutil.ReadAll(tmpFile)
			c.Assume(err, gs.IsNil)
			c.Expect(string(contents), gs.Equals, data)
			c.Expect(output.processMessageCount, gs.Equals, int64(1))
			c.Expect(output.processMessageSamples, gs.Equals, int64(1))
			c.Expect(output.processMessageFailures, gs.Equals, int64(0))
		})

		c.Specify("failure processing data", func() {
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			pack.Message.SetPayload("FAILURE")
			pack.BufferedPack = true
			pack.DelivErrChan = make(chan error, 1)
			inChan <- pack
			close(inChan)
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Assume(err, gs.IsNil)
			c.Expect(output.processMessageFailures, gs.Equals, int64(1))
			err = <-pack.DelivErrChan
			c.Expect(err.Error(), gs.Equals, "failure message")
		})

		c.Specify("user abort processing data", func() {
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			e := errors.New("FATAL: user abort")
			pack.Message.SetPayload("USERABORT")
			inChan <- pack
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err.Error(), gs.Equals, e.Error())
		})

		c.Specify("fatal error processing data", func() {
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			pack.Message.SetPayload("FATAL")
			inChan <- pack
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		c.Specify("fatal error in timer_event", func() {
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			timer <- time.Now()
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err, gs.Not(gs.IsNil))
		})

		c.Specify("fatal error in shutdown timer_event", func() {
			conf.TimerEventOnShutdown = true
			err := output.Init(conf)
			c.Expect(err, gs.IsNil)
			close(inChan)
			err = output.Run(oth.MockOutputRunner, oth.MockHelper)
			c.Expect(err, gs.Not(gs.IsNil))
		})
	})
}

func main() {
	configPath := flag.String("config", filepath.FromSlash("/etc/hekad.toml"),
		"Config file or directory. If directory is specified then all files "+
			"in the directory will be loaded.")
	version := flag.Bool("version", false, "Output version and exit")
	flag.Parse()

	config := &HekadConfig{}
	var err error
	var cpuProfName string
	var memProfName string

	if flag.NFlag() == 0 {
		flag.PrintDefaults()
		os.Exit(0)
	}

	if *version {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	config, err = LoadHekadConfig(*configPath)
	if err != nil {
		log.Fatal("Error reading config: ", err)
	}
	globals, cpuProfName, memProfName := setGlobalConfigs(config)

	if err = os.MkdirAll(globals.BaseDir, 0755); err != nil {
		log.Fatalf("Error creating base_dir %s: %s", config.BaseDir, err)
	}

	if cpuProfName != "" {
		profFile, err := os.Create(cpuProfName)
		if err != nil {
			log.Fatalln(err)
		}
		profFile.Close()
		pprof.StartCPUProfile(profFile)
		defer pprof.StopCPUProfile()
	}

	if memProfName != "" {
		defer func() {
			profFile, err := os.Create(memProfName)
			if err != nil {
				log.Fatalln(err)
			}
			pprof.WriteHeapProfile(profFile)
			profFile.Close()
		}()
	}

	// Set up and load the pipeline configuration and start the daemon.
	pipeconf := pipeline.NewPipelineConfig(globals)
	p, err := os.Open(*configPath)
	fi, err := p.Stat()

	if fi.IsDir() {
		files, _ := ioutil.ReadDir(*configPath)
		for _, f := range files {
			err = pipeconf.LoadFromConfigFile(filepath.Join(*configPath, f.Name()))
		}
	} else {
		err = pipeconf.LoadFromConfigFile(*configPath)
	}

	if err != nil {
		log.Fatal("Error reading config: ", err)
	}
	pipeline.Run(pipeconf)
}

func DecoderSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	pConfig := pipeline.NewPipelineConfig(nil)

	c.Specify("A SandboxDecoder", func() {

		decoder := new(SandboxDecoder)
		decoder.SetPipelineConfig(pConfig)
		conf := decoder.ConfigStruct().(*sandbox.SandboxConfig)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		dRunner := pm.NewMockDecoderRunner(ctrl)

		c.Specify("that uses lpeg and inject_message", func() {
			dRunner.EXPECT().Name().Return("serialize")
			conf.ScriptFilename = "../lua/testsupport/decoder.lua"
			err := decoder.Init(conf)
			c.Assume(err, gs.IsNil)

			c.Specify("decodes simple messages", func() {
				data := "1376389920 debug id=2321 url=example.com item=1"
				decoder.SetDecoderRunner(dRunner)
				pack.Message.SetPayload(data)
				_, err = decoder.Decode(pack)
				c.Assume(err, gs.IsNil)

				c.Expect(pack.Message.GetTimestamp(),
					gs.Equals,
					int64(1376389920000000000))

				c.Expect(pack.Message.GetSeverity(), gs.Equals, int32(7))

				var ok bool
				var value interface{}
				value, ok = pack.Message.GetFieldValue("id")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "2321")

				value, ok = pack.Message.GetFieldValue("url")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "example.com")

				value, ok = pack.Message.GetFieldValue("item")
				c.Expect(ok, gs.Equals, true)
				c.Expect(value, gs.Equals, "1")
				decoder.Shutdown()
			})

			c.Specify("decodes an invalid messages", func() {
				data := "1376389920 bogus id=2321 url=example.com item=1"
				decoder.SetDecoderRunner(dRunner)
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(len(packs), gs.Equals, 0)
				c.Expect(err.Error(), gs.Equals, "Failed parsing: "+data)
				c.Expect(decoder.processMessageFailures, gs.Equals, int64(1))
				decoder.Shutdown()
			})

			c.Specify("Preserves data", func() {
				conf.ScriptFilename = "../lua/testsupport/serialize.lua"
				conf.PreserveData = true
				err := decoder.Init(conf)
				c.Assume(err, gs.IsNil)
				decoder.SetDecoderRunner(dRunner)
				decoder.Shutdown()
				_, err = os.Stat("sandbox_preservation/serialize.data")
				c.Expect(err, gs.IsNil)
				err = os.Remove("sandbox_preservation/serialize.data")
				c.Expect(err, gs.IsNil)
			})
		})

		c.Specify("that only uses write_message", func() {
			conf.ScriptFilename = "../lua/testsupport/write_message_decoder.lua"
			dRunner.EXPECT().Name().Return("write_message")
			err := decoder.Init(conf)
			decoder.SetDecoderRunner(dRunner)
			c.Assume(err, gs.IsNil)

			c.Specify("adds a string field to the message", func() {
				data := "string field scribble"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
				c.Expect(len(packs), gs.Equals, 1)
				c.Expect(packs[0], gs.Equals, pack)
				value, ok := pack.Message.GetFieldValue("scribble")
				c.Expect(ok, gs.IsTrue)
				c.Expect(value.(string), gs.Equals, "foo")
			})

			c.Specify("adds a numeric field to the message", func() {
				data := "num field scribble"
				pack.Message.SetPayload(data)
				packs, err := decoder.Decode(pack)
				c.Expect(err, gs.IsNil)
//.........這裏部分代碼省略.........

func EncoderSpec(c gs.Context) {
	t := new(ts.SimpleT)
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	// NewPipelineConfig sets up Globals which is needed for the
	// pipeline.Prepend*Dir functions to not die during plugin Init().
	pConfig := pipeline.NewPipelineConfig(nil)

	c.Specify("A SandboxEncoder", func() {

		encoder := new(SandboxEncoder)
		encoder.SetPipelineConfig(pConfig)
		conf := encoder.ConfigStruct().(*SandboxEncoderConfig)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
		pack.Message.SetPayload("original")
		pack.Message.SetType("my_type")
		pack.Message.SetPid(12345)
		pack.Message.SetSeverity(4)
		pack.Message.SetHostname("hostname")
		pack.Message.SetTimestamp(54321)
		pack.Message.SetUuid(uuid.NewRandom())
		var (
			result []byte
			err    error
		)

		c.Specify("emits JSON correctly", func() {
			conf.ScriptFilename = "../lua/testsupport/encoder_json.lua"
			err = encoder.Init(conf)
			c.Expect(err, gs.IsNil)

			result, err = encoder.Encode(pack)
			c.Expect(err, gs.IsNil)
			msg := new(message.Message)
			err = json.Unmarshal(result, msg)
			c.Expect(err, gs.IsNil)
			c.Expect(msg.GetTimestamp(), gs.Equals, int64(54321))
			c.Expect(msg.GetPid(), gs.Equals, int32(12345))
			c.Expect(msg.GetSeverity(), gs.Equals, int32(4))
			c.Expect(msg.GetHostname(), gs.Equals, "hostname")
			c.Expect(msg.GetPayload(), gs.Equals, "original")
			c.Expect(msg.GetType(), gs.Equals, "my_type")
		})

		c.Specify("emits text correctly", func() {
			conf.ScriptFilename = "../lua/testsupport/encoder_text.lua"
			err = encoder.Init(conf)
			c.Expect(err, gs.IsNil)

			result, err = encoder.Encode(pack)
			c.Expect(err, gs.IsNil)
			c.Expect(string(result), gs.Equals, "Prefixed original")
		})

		c.Specify("emits protobuf correctly", func() {

			c.Specify("when inject_message is used", func() {
				conf.ScriptFilename = "../lua/testsupport/encoder_protobuf.lua"
				err = encoder.Init(conf)
				c.Expect(err, gs.IsNil)

				result, err = encoder.Encode(pack)
				c.Expect(err, gs.IsNil)

				msg := new(message.Message)
				err = proto.Unmarshal(result, msg)
				c.Expect(err, gs.IsNil)
				c.Expect(msg.GetTimestamp(), gs.Equals, int64(54321))
				c.Expect(msg.GetPid(), gs.Equals, int32(12345))
				c.Expect(msg.GetSeverity(), gs.Equals, int32(4))
				c.Expect(msg.GetHostname(), gs.Equals, "hostname")
				c.Expect(msg.GetPayload(), gs.Equals, "mutated")
				c.Expect(msg.GetType(), gs.Equals, "after")
			})

			c.Specify("when `write_message` is used", func() {
				conf.ScriptFilename = "../lua/testsupport/encoder_writemessage.lua"
				err = encoder.Init(conf)
				c.Expect(err, gs.IsNil)

				result, err = encoder.Encode(pack)
				c.Expect(err, gs.IsNil)

				msg := new(message.Message)
				err = proto.Unmarshal(result, msg)
				c.Expect(err, gs.IsNil)
				c.Expect(msg.GetPayload(), gs.Equals, "mutated payload")
				c.Expect(pack.Message.GetPayload(), gs.Equals, "original")
			})
		})
	})

	c.Specify("cbuf librato encoder", func() {
		encoder := new(SandboxEncoder)
		encoder.SetPipelineConfig(pConfig)
		conf := encoder.ConfigStruct().(*SandboxEncoderConfig)
		supply := make(chan *pipeline.PipelinePack, 1)
		pack := pipeline.NewPipelinePack(supply)
//.........這裏部分代碼省略.........