Golang zlib.NewWriterLevel函数代码示例

func (g *generator) defineSchemaVar() error {
	msg, seg, _ := capnp.NewMessage(capnp.SingleSegment(nil))
	req, _ := schema.NewRootCodeGeneratorRequest(seg)
	fnodes := g.nodes[g.fileID].nodes
	ids := make([]uint64, len(fnodes))
	for i, n := range fnodes {
		ids[i] = n.Id()
	}
	sort.Sort(uint64Slice(ids))
	// TODO(light): find largest object size and use that to allocate list
	nodes, _ := req.NewNodes(int32(len(g.nodes)))
	i := 0
	for _, id := range ids {
		n := g.nodes[id]
		if err := nodes.Set(i, n.Node); err != nil {
			return err
		}
		i++
	}
	var buf bytes.Buffer
	z, _ := zlib.NewWriterLevel(&buf, zlib.BestCompression)
	if err := capnp.NewPackedEncoder(z).Encode(msg); err != nil {
		return err
	}
	if err := z.Close(); err != nil {
		return err
	}
	return renderSchemaVar(g.r, schemaVarParams{
		G:       g,
		FileID:  g.fileID,
		NodeIDs: ids,
		schema:  buf.Bytes(),
	})
}

func (rw *RedisWrapper) SaveRequest(req *ParsedRequest, creatives []*Inventory, timeout int) error {
	conn := rw.redisPool.Get()
	defer conn.Close()
	creativesForRedis := make([]*InventoryForRedis, len(creatives))
	for index, value := range creatives {
		creativesForRedis[index] = &InventoryForRedis{
			AdId:      value.AdId,
			Frequency: value.Frequency,
		}
	}
	req.Creatives = creativesForRedis
	body, err := json.Marshal(*req)
	if err != nil {
		return err
	}
	var buf bytes.Buffer
	zlibWriter, err := zlib.NewWriterLevel(&buf, zlib.BestSpeed)
	if err != nil {
		return err
	}
	if _, err := zlibWriter.Write(body); err != nil {
		return err
	}
	zlibWriter.Close()
	if _, err := conn.Do("setex", rw.configure.RedisCachePrefix+req.Id, timeout, buf.Bytes()); err != nil {
		rw.logger.Warning("redis error: %v", err.Error())
		return err
	}
	return nil
}

// SerializeCompressed serializes a compressed data packet to w and
// returns a WriteCloser to which the literal data packets themselves
// can be written and which MUST be closed on completion. If cc is
// nil, sensible defaults will be used to configure the compression
// algorithm.
func SerializeCompressed(w io.WriteCloser, algo CompressionAlgo, cc *CompressionConfig) (literaldata io.WriteCloser, err error) {
	compressed, err := serializeStreamHeader(w, packetTypeCompressed)
	if err != nil {
		return
	}

	_, err = compressed.Write([]byte{uint8(algo)})
	if err != nil {
		return
	}

	level := DefaultCompression
	if cc != nil {
		level = cc.Level
	}

	var compressor io.WriteCloser
	switch algo {
	case CompressionZIP:
		compressor, err = flate.NewWriter(compressed, level)
	case CompressionZLIB:
		compressor, err = zlib.NewWriterLevel(compressed, level)
	default:
		s := strconv.Itoa(int(algo))
		err = errors.UnsupportedError("Unsupported compression algorithm: " + s)
	}
	if err != nil {
		return
	}

	literaldata = compressedWriteCloser{compressed, compressor}

	return
}

func (o *PrtGenerator) Start(outFilename string, total int, maxProcs int, boundary *BoundaryLocator) {
	o.enclosedsChan = make(chan *EnclosedChunkJob, maxProcs*2)
	o.completeChan = make(chan bool)
	o.total = total
	o.boundary = boundary

	maxProcs = 1
	for i := 0; i < maxProcs; i++ {
		go o.chunkProcessor()
	}

	var openErr os.Error

	o.outFile, openErr = os.Open(outFilename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
	if openErr != nil {
		fmt.Fprintln(os.Stderr, openErr) // TODO: return openErr
		return
	}

	o.w = bufio.NewWriter(o.outFile)
	WriteHeader(o.w, -1, []ChannelDefinition{{"Position", 4, 3, 0}, {"BlockID", 1, 1, 12}})

	var zErr os.Error
	o.zw, zErr = zlib.NewWriterLevel(o.w, zlib.NoCompression)
	if zErr != nil {
		fmt.Fprintln(os.Stderr, zErr) // TODO: return zErr
		return
	}
}

// Returns a zlib-compressed copy of the specified byte array
func sliceCompress(data []byte) []byte {
	var buf bytes.Buffer
	cmp, _ := zlib.NewWriterLevel(&buf, zlib.BestSpeed)
	cmp.Write(data)
	cmp.Close()
	return buf.Bytes()
}

func (f *Ftail) Flush() error {
	if f.buf.Len() <= 0 {
		return nil
	}
	var b bytes.Buffer
	w, err := zlib.NewWriterLevel(&b, zlib.BestCompression)
	if err != nil {
		return err
	}
	row := core.Row{Time: f.lastTime, Pos: f.Pos}
	row.Text = f.buf.String()
	_, err = io.Copy(w, &f.buf)
	if cerr := w.Close(); cerr != nil {
		log.Printf("zlib close err:%s", cerr)
	}
	if err != nil {
		return err
	}
	if b.Len() < f.buf.Len() {
		row.Bin = b.Bytes()
		row.Text = ""
	}
	//log.Printf("text:'%s',bin:'%x', buf.String:%s", row.Text, row.Bin, f.buf.String())
	defer f.buf.Reset()
	if err = f.rec.Put(row); err != nil {
		log.Printf("Flush %s err:%s", f.Pos.Name, err)
	}
	return err
}

//func CompressData(in []byte) (data *bytes.Buffer, err error) {
func CompressData(in []byte) (out []byte, err error) {
	if len(in) == 0 {
		return nil, &AcError{Value: -1, Msg: "CompressData() invalid input: ", Err: err}
	}

	// first let's compress
	data := new(bytes.Buffer)

	zbuf, err := zlib.NewWriterLevel(data, zlib.BestCompression)
	if err != nil {
		return nil, &AcError{Value: -2, Msg: "CompressData().zlib.NewWriterLevel(): ", Err: err}
	}

	n, err := zbuf.Write(in)
	if err != nil || n != len(in) {
		return nil, &AcError{Value: -3, Msg: "CompressData().zlib.Write(): ", Err: err}
	}

	//XXX funny  Flush don't actually flush stuff from zlib into the writer all the time.....
	//zbuf.Flush()
	// XXX let's try...
	zbuf.Close()

	//fmt.Fprintf(os.Stderr, "CompressData(%d B): %d B\n", len(in), data.Len())
	out = data.Bytes()
	return out, nil
}

func compress(s string) string {
	var b bytes.Buffer
	w, _ := zlib.NewWriterLevel(&b, zlib.BestSpeed) // flate.BestCompression
	w.Write([]byte(s))
	w.Close()
	return b.String()
}

func NewPacketCodecZlibLevel(threshold int, level int) (this *PacketCodecZlib) {
	this = new(PacketCodecZlib)
	this.threshold = threshold
	this.level = level
	this.zlibWriter, _ = zlib.NewWriterLevel(nil, level)
	return
}

func (pp *pendingPayload) Generate(hostname string) (err error) {
	var buffer bytes.Buffer

	// Begin with the nonce
	if _, err = buffer.Write([]byte(pp.nonce)); err != nil {
		return
	}

	var compressor *zlib.Writer
	if compressor, err = zlib.NewWriterLevel(&buffer, 3); err != nil {
		return
	}

	// Append all the events
	for _, event := range pp.events[pp.ack_events:] {
		// Add host field
		event.Event["host"] = hostname
		if err = pp.bufferJdatDataEvent(compressor, event); err != nil {
			return
		}
	}

	compressor.Close()

	pp.payload = buffer.Bytes()
	pp.payload_start = pp.ack_events

	return
}

func (o *PrtGenerator) Start(outFilename string, total int, maxProcs int, boundary *BoundaryLocator) error {
	o.enclosedsChan = make(chan *EnclosedChunkJob, maxProcs*2)
	o.completeChan = make(chan bool)
	o.total = total
	o.boundary = boundary

	maxProcs = 1
	for i := 0; i < maxProcs; i++ {
		go o.chunkProcessor()
	}

	var openErr error

	o.outFile, openErr = os.Create(outFilename)
	if openErr != nil {
		return openErr
	}

	o.w = bufio.NewWriter(o.outFile)
	WriteHeader(o.w, -1, []ChannelDefinition{{"Position", 4, 3, 0}, {"BlockID", 1, 1, 12}})

	var zErr error
	o.zw, zErr = zlib.NewWriterLevel(o.w, zlib.NoCompression)
	if zErr != nil {
		return zErr
	}

	return nil
}

func (h Handler) printResponse(status int, header map[string]string, content []byte) {
	headerEncoded := encodeData(header)

	h.response.WriteHeader(200)
	h.response.Header().Set("Content-Type", "image/gif")

	compressed := false
	if contentType, ok := header["content-type"]; ok {
		if strings.HasPrefix(contentType, "text/") || strings.HasPrefix(contentType, "application/json") || strings.HasPrefix(contentType, "application/javascript") {
			compressed = true
		}
	}

	if compressed {
		h.response.Write([]byte("1"))
		w, err := zlib.NewWriterLevel(h.response, zlib.BestCompression)
		if err != nil {
			h.context.Criticalf("zlib.NewWriterDict(h.response, zlib.BestCompression, nil) Error: %v", err)
			return
		}
		defer w.Close()
		binary.Write(w, binary.BigEndian, uint32(status))
		binary.Write(w, binary.BigEndian, uint32(len(headerEncoded)))
		binary.Write(w, binary.BigEndian, uint32(len(content)))
		w.Write(headerEncoded)
		w.Write(content)
	} else {
		h.response.Write([]byte("0"))
		binary.Write(h.response, binary.BigEndian, uint32(status))
		binary.Write(h.response, binary.BigEndian, uint32(len(headerEncoded)))
		binary.Write(h.response, binary.BigEndian, uint32(len(content)))
		h.response.Write(headerEncoded)
		h.response.Write(content)
	}
}

func newLumberjackClient(
	conn TransportClient,
	compressLevel int,
	maxWindowSize int,
	timeout time.Duration,
	beat string,
) (*lumberjackClient, error) {

	// validate by creating and discarding zlib writer with configured level
	if compressLevel > 0 {
		tmp := bytes.NewBuffer(nil)
		w, err := zlib.NewWriterLevel(tmp, compressLevel)
		if err != nil {
			return nil, err
		}
		w.Close()
	}

	encodedBeat, err := json.Marshal(beat)
	if err != nil {
		return nil, err
	}

	return &lumberjackClient{
		TransportClient: conn,
		windowSize:      defaultStartMaxWindowSize,
		timeout:         timeout,
		maxWindowSize:   maxWindowSize,
		compressLevel:   compressLevel,
		beat:            encodedBeat,
	}, nil
}

func (this *PacketCodecZlib) Encode(writer io.Writer, util []byte, packet Packet) (err error) {
	buffer := new(bytes.Buffer)
	err = this.codec.Encode(buffer, util, packet)
	if err != nil {
		return
	}
	if raw, ok := packet.(PacketRaw); ok && raw.Raw() {
		_, err = buffer.WriteTo(writer)
	} else if buffer.Len() < this.threshold {
		err = WriteVarInt(writer, util, 0)
		if err != nil {
			return
		}
		_, err = buffer.WriteTo(writer)
	} else {
		err = WriteVarInt(writer, util, buffer.Len())
		if err != nil {
			return
		}
		var zlibWriter io.WriteCloser
		zlibWriter, err = zlib.NewWriterLevel(writer, this.level)
		if err != nil {
			return
		}
		_, err = buffer.WriteTo(zlibWriter)
		if err != nil {
			return
		}
		err = zlibWriter.Close()
	}
	return
}

func (c *conn) packRequest(r *http.Request) (*http.Request, error) {
	buf := &bytes.Buffer{}
	zbuf, err := zlib.NewWriterLevel(buf, zlib.BestCompression)
	if err != nil {
		return nil, fmt.Errorf("conn.packRequest(zlib.NewWriterLevel)>%s", err)
	}
	url := c.url + r.URL.String()
	urlhex := make([]byte, hex.EncodedLen(len(url)))
	hex.Encode(urlhex, []byte(url))
	fmt.Fprintf(zbuf, "url=%s", urlhex)
	fmt.Fprintf(zbuf, "&method=%s", hex.EncodeToString([]byte(r.Method)))
	if c.ps.password != "" {
		fmt.Fprintf(zbuf, "&password=%s", c.ps.password)
	}
	fmt.Fprint(zbuf, "&headers=")
	for k, v := range r.Header {
		fmt.Fprint(zbuf, hex.EncodeToString([]byte(fmt.Sprintf("%s:%s\r\n", k, v[0]))))
	}
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, fmt.Errorf("conn.packRequest(ioutil.ReadAll(r.Body))>%s", err)
	}
	payload := hex.EncodeToString(body)
	fmt.Fprintf(zbuf, "&payload=%s", payload)
	zbuf.Close()
	req, err := http.NewRequest("POST", c.ps.path, buf)
	if err != nil {
		return nil, fmt.Errorf("conn.packRequest(http.NewRequest)>%s", err)
	}
	req.Host = c.ps.appid[rand.Intn(len(c.ps.appid))] + ".appspot.com"
	req.URL.Scheme = "http"
	return req, nil
}