Golang bufio.NewWriterSize函数代码示例

func CommandExecutor(app *global.AssemblyWithComponents) (action.Result, error) {
	var e exec.OsExecutor
	var commandWords []string
	appName := ""
	commandWords = strings.Fields(app.Command)
	log.Debug("Command Executor entry: %s\n", app)
	megam_home, ckberr := config.GetString("megam_home")
	if ckberr != nil {
		return nil, ckberr
	}
	pair, perr := global.ParseKeyValuePair(app.Inputs, "domain")
	if perr != nil {
		log.Error("Failed to get the domain value : %s", perr)
	}

	appName = app.Name + "." + pair.Value

	basePath := megam_home + "logs"
	dir := path.Join(basePath, appName)

	fileOutPath := path.Join(dir, appName+"_out")
	fileErrPath := path.Join(dir, appName+"_err")
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		log.Info("Creating directory: %s\n", dir)
		if errm := os.MkdirAll(dir, 0777); errm != nil {
			return nil, errm
		}
	}
	// open output file
	fout, outerr := os.OpenFile(fileOutPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if outerr != nil {
		return nil, outerr
	}
	defer fout.Close()
	// open Error file
	ferr, errerr := os.OpenFile(fileErrPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if errerr != nil {
		return nil, errerr
	}
	defer ferr.Close()

	foutwriter := bufio.NewWriterSize(fout, 1)
	ferrwriter := bufio.NewWriterSize(ferr, 1)
	log.Debug(commandWords)
	log.Debug("Length: %s", len(commandWords))

	defer ferrwriter.Flush()
	defer foutwriter.Flush()

	if len(commandWords) > 0 {
		if err := e.Execute(commandWords[0], commandWords[1:], nil, foutwriter, ferrwriter); err != nil {
			return nil, err
		}
	}

	return &app, nil
}

func WriteLogsThread(filename string) {
	log.Infof("Started write logs thread to file=%s", filename)

	reopenTick := time.Tick(time.Second * 10)

	fp, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	var wr *bufio.Writer

	if err != nil {
		log.Errorf("Could not open %s: %s", filename, err.Error())
	} else {
		wr = bufio.NewWriterSize(fp, 65536)
	}

	for {
		select {
		case <-reopenTick:
			if fp != nil {
				fp.Close()
			}

			fp, err = os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
			if err != nil {
				log.Warnf("Could not reopen %s: %s", err.Error())
				wr = nil
				fp = nil
			} else {
				wr = bufio.NewWriterSize(fp, 65536)
			}
		case ev := <-rqhLog:
			l := len(rqhLog)
			evs := make([]*FinishResult, 0, l+1)
			evs = append(evs, ev)

			for i := 0; i < l; i++ {
				evs = append(evs, <-rqhLog)
			}

			if wr != nil {
				encoder := json.NewEncoder(wr)

				for _, e := range evs {
					if err = encoder.Encode(e); err != nil {
						log.Errorf("Could not write to %s: %s", filename, err.Error())
					}
				}

				if err = wr.Flush(); err != nil {
					log.Errorf("Could not flush contents to %s: %s", filename, err.Error())
				}
			} else {
				log.Errorf("Failed to write %d events to rqh log because file %s could not be opened", len(evs), filename)
			}
		}

	}
}

// handleConnection wraps the in and out connection endpoints for transfer between them.
func handleConnection(clientConn net.Conn, serverConn net.Conn, txHandlers []func([]byte), rxHandlers []func([]byte)) {
	clientReader := bufio.NewReaderSize(clientConn, 8192)
	clientWriter := bufio.NewWriterSize(clientConn, 8192)

	serverReader := bufio.NewReaderSize(serverConn, 8192)
	serverWriter := bufio.NewWriterSize(serverConn, 8192)

	go pipeReaderToWriter(clientReader, serverWriter, txHandlers)
	go pipeReaderToWriter(serverReader, clientWriter, rxHandlers)
}

func (cmd *cmdSync) SendPSyncCmd(master, passwd string) (pipe.Reader, int64) {
	c := openNetConn(master, passwd)
	br := bufio.NewReaderSize(c, ReaderBufferSize)
	bw := bufio.NewWriterSize(c, WriterBufferSize)

	runid, offset, wait := sendPSyncFullsync(br, bw)
	log.Infof("psync runid = %s offset = %d, fullsync", runid, offset)

	var nsize int64
	for nsize == 0 {
		select {
		case nsize = <-wait:
			if nsize == 0 {
				log.Info("+")
			}
		case <-time.After(time.Second):
			log.Info("-")
		}
	}

	piper, pipew := pipe.NewSize(ReaderBufferSize)

	go func() {
		defer pipew.Close()
		p := make([]byte, 8192)
		for rdbsize := int(nsize); rdbsize != 0; {
			rdbsize -= iocopy(br, pipew, p, rdbsize)
		}
		for {
			n, err := cmd.PSyncPipeCopy(c, br, bw, offset, pipew)
			if err != nil {
				log.PanicErrorf(err, "psync runid = %s, offset = %d, pipe is broken", runid, offset)
			}
			offset += n
			for {
				time.Sleep(time.Second)
				c = openNetConnSoft(master, passwd)
				if c != nil {
					log.Infof("psync reopen connection, offset = %d", offset)
					break
				} else {
					log.Infof("psync reopen connection, failed")
				}
			}
			authPassword(c, passwd)
			br = bufio.NewReaderSize(c, ReaderBufferSize)
			bw = bufio.NewWriterSize(c, WriterBufferSize)
			sendPSyncContinue(br, bw, runid, offset)
		}
	}()
	return piper, nsize
}

func Benchmark_PubFourQueueSub(b *testing.B) {
	b.StopTimer()
	s := runBenchServer()
	c := createClientConn(b, "localhost", PERF_PORT)
	doDefaultConnect(b, c)
	sendProto(b, c, "SUB foo group1 1\r\n")
	sendProto(b, c, "SUB foo group1 2\r\n")
	sendProto(b, c, "SUB foo group1 3\r\n")
	sendProto(b, c, "SUB foo group1 4\r\n")
	bw := bufio.NewWriterSize(c, defaultSendBufSize)
	sendOp := []byte(fmt.Sprintf("PUB foo 2\r\nok\r\n"))
	ch := make(chan bool)
	expected := len("MSG foo 1 2\r\nok\r\n") * b.N
	go drainConnection(b, c, ch, expected)
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		_, err := bw.Write(sendOp)
		if err != nil {
			b.Fatalf("Received error on PUB write: %v\n", err)
		}
	}
	err := bw.Flush()
	if err != nil {
		b.Fatalf("Received error on FLUSH write: %v\n", err)
	}

	// Wait for connection to be drained
	<-ch

	b.StopTimer()
	c.Close()
	s.Shutdown()
}

func (c *ClientV2) SetOutputBufferSize(desiredSize int) error {
	var size int

	switch {
	case desiredSize == -1:
		// effectively no buffer (every write will go directly to the wrapped net.Conn)
		size = 1
	case desiredSize == 0:
		// do nothing (use default)
	case desiredSize >= 64 && desiredSize <= int(c.context.nsqd.options.MaxOutputBufferSize):
		size = desiredSize
	default:
		return errors.New(fmt.Sprintf("output buffer size (%d) is invalid", desiredSize))
	}

	if size > 0 {
		c.Lock()
		defer c.Unlock()
		c.OutputBufferSize = size
		err := c.Writer.Flush()
		if err != nil {
			return err
		}
		c.Writer = bufio.NewWriterSize(c.Conn, size)
	}

	return nil
}

// NewBufferedWriter allocates and returns a BufferedWriter with an internal
// buffer of MaxBlockSize bytes.  If an error occurs writing a block to w, all
// future writes will fail with the same error.  After all data has been
// written, the client should call the Flush method to guarantee all data has
// been forwarded to the underlying io.Writer.
func NewBufferedWriter(w io.Writer) *BufferedWriter {
	_w := NewWriter(w).(*writer)
	return &BufferedWriter{
		w:  _w,
		bw: bufio.NewWriterSize(_w, MaxBlockSize),
	}
}

// This will process a disconnect when reconnect is allowed.
// The lock should not be held on entering this function.
func (nc *Conn) processReconnect() {

	nc.mu.Lock()
	defer nc.mu.Unlock()

	if !nc.IsClosed() {
		// If we are already in the proper state, just return.
		if nc.status == RECONNECTING {
			return
		}
		nc.status = RECONNECTING
		if nc.conn != nil {
			nc.bw.Flush()
			nc.conn.Close()
		}
		nc.conn = nil
		nc.kickFlusher()

		// FIXME(dlc) - We have an issue here if we have
		// outstanding flush points (pongs) and they were not
		// sent out, but are still in the pipe.

		// Create a pending buffer to underpin the bufio Writer while
		// we are reconnecting.
		nc.pending = &bytes.Buffer{}
		nc.bw = bufio.NewWriterSize(nc.pending, defaultPendingSize)
		nc.err = nil
		go nc.doReconnect()
	}
	// Perform appropriate callback if needed for a disconnect.
	dcb := nc.Opts.DisconnectedCB
	if dcb != nil {
		go dcb(nc)
	}
}

func NewClientV2(conn net.Conn) *ClientV2 {
	var identifier string
	if conn != nil {
		identifier, _, _ = net.SplitHostPort(conn.RemoteAddr().String())
	}

	return &ClientV2{
		Conn: conn,
		// ReadyStateChan has a buffer of 1 to guarantee that in the event
		// there is a race the state update is not lost
		ReadyStateChan:  make(chan int, 1),
		ExitChan:        make(chan int),
		ConnectTime:     time.Now(),
		ShortIdentifier: identifier,
		LongIdentifier:  identifier,
		Reader:          bufio.NewReaderSize(conn, 16*1024),
		Writer:          bufio.NewWriterSize(conn, 16*1024),
		State:           nsq.StateInit,
		SubEventChan:    make(chan *Channel, 1),

		// heartbeats are client configurable but default to 30s
		HeartbeatInterval:   nsqd.options.clientTimeout / 2,
		HeartbeatUpdateChan: make(chan time.Duration, 1),
	}
}

// newFileWriter returns a prepared fileWriter for the driver and path. This
// could be considered similar to an "open" call on a regular filesystem.
func newFileWriter(driver storagedriver.StorageDriver, path string) (*bufferedFileWriter, error) {
	fw := fileWriter{
		driver: driver,
		path:   path,
	}

	if fi, err := driver.Stat(path); err != nil {
		switch err := err.(type) {
		case storagedriver.PathNotFoundError:
			// ignore, offset is zero
		default:
			return nil, err
		}
	} else {
		if fi.IsDir() {
			return nil, fmt.Errorf("cannot write to a directory")
		}

		fw.size = fi.Size()
	}

	buffered := bufferedFileWriter{
		fileWriter: fw,
	}
	buffered.bw = bufio.NewWriterSize(&buffered.fileWriter, fileWriterBufferSize)

	return &buffered, nil
}

func (l *BinLog) openNewLogFile() error {
	var err error
	lastName := l.getLogFile()

	logPath := path.Join(l.path, lastName)
	if l.logFile, err = os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY, 0666); err != nil {
		log.Error("open new logfile error %s", err.Error())
		return err
	}

	if l.cfg.MaxFileNum > 0 && len(l.logNames) == l.cfg.MaxFileNum {
		l.purge(1)
	}

	l.logNames = append(l.logNames, lastName)

	if l.logWb == nil {
		l.logWb = bufio.NewWriterSize(l.logFile, 1024)
	} else {
		l.logWb.Reset(l.logFile)
	}

	if err = l.flushIndex(); err != nil {
		return err
	}

	return nil
}

// rotateFile closes the syncBuffer's file and starts a new one.
func (sb *syncBuffer) rotateFile(now time.Time) error {
	if sb.file != nil {
		sb.Flush()
		sb.file.Close()
	}
	var err error
	sb.file, _, err = create(severityName[sb.sev], now)
	sb.nbytes = 0
	if err != nil {
		return err
	}

	sb.Writer = bufio.NewWriterSize(sb.file, bufferSize)
	_, month, day := now.Date()
	sb.createdDate = fmt.Sprintf("%02d%02d", month, day)

	// Write header.
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "Log file created at: %s\n", now.Format("2006/01/02 15:04:05"))
	fmt.Fprintf(&buf, "Running on machine: %s\n", host)
	fmt.Fprintf(&buf, "Binary: Built with %s %s for %s/%s\n", runtime.Compiler, runtime.Version(), runtime.GOOS, runtime.GOARCH)
	fmt.Fprintf(&buf, "Log line format: [DIEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg\n")
	n, err := sb.file.Write(buf.Bytes())
	sb.nbytes += uint64(n)
	return err
}

// NewSuperBlock new a super block struct.
func NewSuperBlock(file string) (b *SuperBlock, err error) {
	b = &SuperBlock{}
	b.file = file
	b.buf = make([]byte, NeedleMaxSize)
	if b.w, err = os.OpenFile(file, os.O_WRONLY|os.O_CREATE, 0664); err != nil {
		log.Errorf("os.OpenFile(\"%s\", os.O_WRONLY|os.O_CREATE, 0664) error(%v)", file, err)
		return
	}
	b.bw = bufio.NewWriterSize(b.w, NeedleMaxSize)
	if b.r, err = os.OpenFile(file, os.O_RDONLY, 0664); err != nil {
		log.Errorf("os.OpenFile(\"%s\", os.O_RDONLY, 0664) error(%v)", file, err)
		goto failed
	}
	if err = b.init(); err != nil {
		log.Errorf("block: %s init error(%v)", file, err)
		goto failed
	}
	return
failed:
	if b.w != nil {
		b.w.Close()
	}
	if b.r != nil {
		b.r.Close()
	}
	return
}

func newAsyncConnHdl(spec *ConnectionSpec) (async *asyncConnHdl, err Error) {
	//	here := "newAsynConnHDL";
	connHdl, err := newConnHdl(spec)
	if err == nil && connHdl != nil {
		async = new(asyncConnHdl)
		if async != nil {
			async.super = connHdl
			//			var e error
			async.writer = bufio.NewWriterSize(connHdl.conn, spec.wBufSize)

			async.pendingReqs = make(chan asyncReqPtr, spec.reqChanCap)
			async.pendingResps = make(chan asyncReqPtr, spec.rspChanCap)
			async.faults = make(chan asyncReqPtr, spec.reqChanCap) // not sure about sizing here ...

			async.reqProcCtl = make(workerCtl)
			async.rspProcCtl = make(workerCtl)
			async.heartbeatCtl = make(workerCtl)
			async.managerCtl = make(workerCtl)

			async.feedback = make(chan workerStatus)
			async.shutdown = make(chan bool, 1)

			return
		}
	}
	// fall through here on errors only
	if debug() {
		log.Println("Error creating asyncConnHdl: ", err)
		//		err =  os.NewError("Error creating asyncConnHdl");
	}
	return nil, err
}

// Run starts the pipeline.
func (p *Pipeline) Run() (int64, error) {
	last := p.input
	for _, f := range p.filters {
		log.Printf("Link %v -> %v", last, f)
		if err := f.Link(last); err != nil {
			return 0, err
		}
		last = f
	}
	buf := bufio.NewWriterSize(p.output, *outputBuffer)
	n, err := io.Copy(buf, last)
	if err != nil {
		return n, fmt.Errorf("Couldn't pipe data: %s", err)
	}
	log.Print("copied")
	if err := buf.Flush(); err != nil {
		return n, fmt.Errorf("Couldn't flush data: %s", err)
	}
	log.Print("flushed")
	if err := p.output.Close(); err != nil {
		return n, fmt.Errorf("Couldn't close pipeline: %s", err)
	}
	log.Print("closed")
	return n, nil
}