Golang PipelinePack.Recycle方法代码示例

func (this *SkyOutput) Run(r engine.OutputRunner, h engine.PluginHelper) error {
	var (
		ok      = true
		pack    *engine.PipelinePack
		inChan  = r.InChan()
		globals = engine.Globals()
		project = h.Project(this.project)
	)

LOOP:
	for ok {
		select {
		case pack, ok = <-inChan:
			if !ok {
				break LOOP
			}

			if globals.Debug {
				globals.Println(*pack)
			}

			this.feedSky(project, pack)
			pack.Recycle()
		}
	}

	return nil
}

func (this *DebugOutput) Run(r engine.OutputRunner, h engine.PluginHelper) error {
	var (
		globals = engine.Globals()
		pack    *engine.PipelinePack
		ok      = true
		inChan  = r.InChan()
	)

LOOP:
	for ok {
		select {
		case pack, ok = <-inChan:
			if !ok {
				break LOOP
			}

			if !this.blackhole {
				globals.Println(*pack)
			}

			pack.Recycle()
		}
	}

	return nil
}

func (this *EsFilter) Run(r engine.FilterRunner, h engine.PluginHelper) error {
	var (
		globals = engine.Globals()
		pack    *engine.PipelinePack
		ok      = true
		count   = 0
		inChan  = r.InChan()
	)

LOOP:
	for ok {
		select {
		case pack, ok = <-inChan:
			if !ok {
				break LOOP
			}

			if globals.Debug {
				globals.Println(*pack)
			}

			if this.handlePack(pack, h.Project(pack.Project)) {
				count += 1
				r.Inject(pack)
			} else {
				pack.Recycle()
			}
		}
	}

	globals.Printf("[%s]Total filtered: %d", r.Name(), count)

	return nil
}

func (this *CardinalityOutput) Run(r engine.OutputRunner, h engine.PluginHelper) error {
	var (
		pack   *engine.PipelinePack
		ok     = true
		inChan = r.InChan()
	)

	h.RegisterHttpApi("/card/{key}", func(w http.ResponseWriter,
		req *http.Request, params map[string]interface{}) (interface{}, error) {
		return this.handleHttpRequest(w, req, params)
	}).Methods("GET", "PUT")

LOOP:
	for ok {
		select {
		case pack, ok = <-inChan:
			if !ok {
				break LOOP
			}

			if pack.CardinalityKey != "" && pack.CardinalityData != nil {
				this.counters.Add(pack.CardinalityKey, pack.CardinalityData)
			}

			pack.Recycle()
		}
	}

	// before we quit, dump counters
	if this.checkpoint != "" {
		this.counters.Dump(this.checkpoint)
	}

	return nil
}

func (this *AlarmOutput) Run(r engine.OutputRunner, h engine.PluginHelper) error {
	var (
		pack       *engine.PipelinePack
		reloadChan = make(chan interface{})
		ok         = true
		inChan     = r.InChan()
	)

	for name, project := range this.projects {
		go this.runSendAlarmsWatchdog(h.Project(name), project)
	}

	// start all the workers
	goAhead := make(chan bool)
	for _, project := range this.projects {
		for _, worker := range project.workers {
			go worker.run(h, goAhead)
			<-goAhead // in case of race condition with worker.inject
		}
	}

	observer.Subscribe(engine.RELOAD, reloadChan)

LOOP:
	for ok {
		select {
		case <-reloadChan:
			// TODO

		case pack, ok = <-inChan:
			if !ok {
				break LOOP
			}

			this.handlePack(pack, h)
			pack.Recycle()
		}
	}

	close(this.stopChan)

	// all the workers cleanup
	for _, project := range this.projects {
		for _, worker := range project.workers {
			worker.cleanup()
		}
	}

	for _, project := range this.projects {
		close(project.emailChan)
	}

	return nil
}

func (this *SelfSysInput) Run(r engine.InputRunner, h engine.PluginHelper) error {
	var (
		globals    = engine.Globals()
		stats      = newSysStat()
		inChan     = r.InChan()
		pack       *engine.PipelinePack
		jsonString string
		err        error
		stopped    = false
	)

	for !stopped {
		select {
		case <-this.stopChan:
			stopped = true

		case <-r.Ticker():
			// same effect as sleep
		}

		if stopped {
			break
		}

		stats.gatherStats()
		jsonString, err = stats.jsonString()
		if err != nil {
			globals.Println(err)
			continue
		}

		pack = <-inChan
		if err = pack.Message.FromLine(fmt.Sprintf("als,%d,%s",
			time.Now().Unix(), jsonString)); err != nil {
			globals.Printf("invalid sys stat: %s\n", jsonString)

			pack.Recycle()
			continue
		}

		pack.Project = "als"
		pack.Ident = this.ident
		pack.EsIndex = "fun_als"
		pack.EsType = "sys"
		r.Inject(pack)
	}

	return nil
}

func (this *EsBufferFilter) Run(r engine.FilterRunner, h engine.PluginHelper) error {
	var (
		pack    *engine.PipelinePack
		ok      = true
		globals = engine.Globals()
		inChan  = r.InChan()
	)

	for _, worker := range this.wokers {
		go worker.run(r, h)
	}

LOOP:
	for ok {
		select {
		case pack, ok = <-inChan:
			if !ok {
				break LOOP
			}

			if globals.Debug {
				globals.Println(*pack)
			}

			this.handlePack(pack)
			pack.Recycle()
		}
	}

	total := 0
	for _, worker := range this.wokers {
		total += worker.summary.N
		worker.flush(r, h)
	}

	// all workers will get notified and stop running
	close(this.stopChan)

	globals.Printf("[%s]Total filtered: %d", r.Name(), total)

	return nil
}

func (this *NetSenderOutput) Run(r engine.OutputRunner, h engine.PluginHelper) error {
	var (
		pack   *engine.PipelinePack
		ok     = true
		inChan = r.InChan()
	)

LOOP:
	for ok {
		select {
		case pack, ok = <-inChan:
			if !ok {
				break LOOP
			}

			this.target(pack).send(pack)
			pack.Recycle()
		}
	}

	return nil
}

func (this *CardinalityFilter) Run(r engine.FilterRunner,
	h engine.PluginHelper) error {
	var (
		pack   *engine.PipelinePack
		ok     = true
		inChan = r.InChan()
	)

LOOP:
	for ok {
		select {
		case pack, ok = <-inChan:
			if !ok {
				break LOOP
			}

			this.handlePack(r, h, pack)
			pack.Recycle()
		}
	}

	return nil
}

func (this *ArchiveInput) doRunSingleLogfile(path string) {
	reader := als.NewAlsReader(path)
	if e := reader.Open(); e != nil {
		panic(e)
	}

	defer func() {
		reader.Close()
		this.workersWg.Done()
		atomic.AddInt32(&this.leftN, -1)

		<-this.workerNChan // release the lock
	}()

	var (
		line    []byte
		lineN   int
		inChan  = this.runner.InChan()
		err     error
		project = this.h.Project(this.project)
		pack    *engine.PipelinePack
		globals = engine.Globals()
	)

	for !this.stopping {
		line, err = reader.ReadLine()
		switch err {
		case nil:
			lineN += 1
			atomic.AddInt64(&this.lineN, 1)
			if globals.Verbose && lineN == 1 {
				project.Printf("[%s]started\n", path)
			}

			pack = <-inChan
			if err = pack.Message.FromLine(string(line)); err != nil {
				if project.ShowError && err != als.ErrEmptyLine {
					project.Printf("[%s]%v: %s", path, err, string(line))
				}

				pack.Recycle()
				continue
			}

			pack.Ident = this.ident
			pack.Project = this.project
			pack.Logfile.SetPath(path)
			if globals.Debug {
				globals.Println(*pack)
			}
			this.runner.Inject(pack)

		case io.EOF:
			if globals.Verbose {
				project.Printf("[%s]done, lines: %d\n", path, lineN)
			}

			this.chkpnt.Put(path)
			this.chkpnt.Dump()

			return

		default:
			// unknown error
			panic(err)
		}
	}

}

func (this *EsOutput) Run(r engine.OutputRunner, h engine.PluginHelper) error {
	var (
		pack         *engine.PipelinePack
		reloadChan   = make(chan interface{})
		ok           = true
		globals      = engine.Globals()
		inChan       = r.InChan()
		reportTicker = time.NewTicker(this.reportInterval)
	)

	this.indexer = core.NewBulkIndexer(this.bulkMaxConn)
	this.indexer.BulkMaxDocs = this.bulkMaxDocs
	this.indexer.BulkMaxBuffer = this.bulkMaxBuffer

	// start the bulk indexer
	this.indexer.Run(this.stopChan)

	defer reportTicker.Stop()

	observer.Subscribe(engine.RELOAD, reloadChan)

LOOP:
	for ok {
		select {
		case <-this.stopChan:
			ok = false

		case <-reportTicker.C:
			this.showPeriodicalStats()

		case <-reloadChan:
			// TODO

		case <-time.After(this.flushInterval):
			this.indexer.Flush()

		case pack, ok = <-inChan:
			if !ok {
				break LOOP
			}

			if globals.Debug {
				globals.Println(*pack)
			}

			this.feedEs(h.Project(pack.Project), pack)
			pack.Recycle()
		}
	}

	engine.Globals().Printf("[%s]Total output to ES: %d", r.Name(), this.totalN)

	// before shutdown, flush again
	if globals.Verbose {
		engine.Globals().Println("Waiting for ES flush...")
	}
	this.indexer.Flush()
	if globals.Verbose {
		engine.Globals().Println("ES flushed")
	}

	// let indexer stop
	this.stopChan <- true

	return nil
}