Golang encoding.Encoding类代码示例

func load(direction string, enc encoding.Encoding) ([]byte, []byte, Transcoder, error) {
	basename, ext, count := "", "", 0
	for _, tf := range testdataFiles {
		if tf.enc == enc {
			basename, ext = tf.basename, tf.ext
			count++
		}
	}
	if count != 1 {
		if count == 0 {
			return nil, nil, nil, fmt.Errorf("no testdataFiles for %s", enc)
		}
		return nil, nil, nil, fmt.Errorf("too many testdataFiles for %s", enc)
	}
	dstFile := fmt.Sprintf("testdata/%s-%s.txt", basename, ext)
	srcFile := fmt.Sprintf("testdata/%s-utf-8.txt", basename)
	var coder Transcoder = encoding.ReplaceUnsupported(enc.NewEncoder())
	if direction == "Decode" {
		dstFile, srcFile = srcFile, dstFile
		coder = enc.NewDecoder()
	}
	dst, err := ioutil.ReadFile(dstFile)
	if err != nil {
		return nil, nil, nil, err
	}
	src, err := ioutil.ReadFile(srcFile)
	if err != nil {
		return nil, nil, nil, err
	}
	return dst, src, coder, nil
}

func encodeText(e encoding.Encoding, text string) []byte {
	res, _, err := transform.Bytes(e.NewEncoder(), []byte(text))
	if err != nil {
		panic(err)
	}
	return res
}

// NewReader returns a reader which decode from the given encoding, to utf8.
//
// If enc is nil, then only an utf8-enforcing replacement reader
// (see http://godoc.org/code.google.com/p/go.text/encoding#pkg-variables)
// is used.
func NewReader(r io.Reader, enc encoding.Encoding) io.Reader {
	if enc == nil || enc == encoding.Replacement {
		return transform.NewReader(r, encoding.Replacement.NewEncoder())
	}
	return transform.NewReader(r,
		transform.Chain(enc.NewDecoder(), encoding.Replacement.NewEncoder()))
}

func ConvTo(b []byte, e encoding.Encoding) (result []byte, err error) {
	w := new(bytes.Buffer)
	writer := transform.NewWriter(w, e.NewEncoder())
	defer writer.Close()

	if _, err = writer.Write(b); err != nil {
		return
	}
	return w.Bytes(), nil
}

func verifyFromUTF(enc encoding.Encoding, b byte, r rune) {

	encoder := enc.NewEncoder()

	out := make([]byte, 6)
	utf := make([]byte, utf8.RuneLen(r))
	utf8.EncodeRune(utf, r)

	ndst, nsrc, err := encoder.Transform(out, utf, true)
	So(err, ShouldBeNil)
	So(nsrc, ShouldEqual, len(utf))
	So(ndst, ShouldEqual, 1)
	So(b, ShouldEqual, out[0])
}

// Convert unicode text to ASCII text
// using specific codepage mapping.
func convertUnicodeToAscii(text string,
	codepage encoding.Encoding) []byte {
	b := []byte(text)
	// fmt.Printf("Text length: %d\n", len(b))
	var buf bytes.Buffer
	if codepage == nil {
		codepage = charmap.Windows1252
	}
	w := transform.NewWriter(&buf, codepage.NewEncoder())
	defer w.Close()
	w.Write(b)
	// fmt.Printf("Buffer length: %d\n", len(buf.Bytes()))
	return buf.Bytes()
}

func GetUTF8HtmlTitle(str string) string {
	var e encoding.Encoding
	var name string

	e, name, _ = charset.DetermineEncoding([]byte(str), "text/html")
	if name == "windows-1252" {
		e, name, _ = charset.DetermineEncoding([]byte(str), "text/html;charset=gbk")
	}
	r := transform.NewReader(strings.NewReader(str), e.NewDecoder())
	if b, err := ioutil.ReadAll(r); err != nil {
		return ""
	} else {
		return getHtmlTitle(string(b))
	}
	return ""
}

func verifyToUTF(enc encoding.Encoding, b byte, r rune) {
	decoder := enc.NewDecoder()

	out := make([]byte, 6)
	nat := []byte{b}
	utf := make([]byte, utf8.RuneLen(r))
	utf8.EncodeRune(utf, r)

	ndst, nsrc, err := decoder.Transform(out, nat, true)
	So(err, ShouldBeNil)
	So(nsrc, ShouldEqual, 1)
	if !bytes.Equal(utf, out[:ndst]) {
		Printf("UTF expected %v, but got %v for %x\n", utf, out, b)
	}
	So(bytes.Equal(utf, out[:ndst]), ShouldBeTrue)
}

// There might be a shortcut when stream is not required, comparing to StreamFromResponse().
func TextFromResponse(response *http.Response) (text, textType string, err error) {
	charset, textType, err := CharsetFromContentType(response.Header.Get("Content-Type"))
	if err != nil {
		return
	}

	body := getBodyStream(response)

	var data []byte
	var stream io.Reader
	var encoding encoding.Encoding
	if (len(charset) == 0) && (textType == "html") {
		data, err = ioutil.ReadAll(body)
		if err != nil {
			return
		}
		charset = DetectCharset(data)
		if encoding, err = GetEncoding(charset); err != nil {
			return
		}
		// No encoding, it assumed as UTF-8.
		if encoding == nil {
			text = string(data)
			return
		}

		stream = transform.NewReader(bytes.NewReader(data), encoding.NewDecoder())
	} else {
		if encoding, err = GetEncoding(charset); err != nil {
			return
		}
		if encoding != nil {
			stream = transform.NewReader(body, encoding.NewDecoder())
		} else {
			stream = body
		}
	}

	if data, err = ioutil.ReadAll(stream); err == nil {
		text = string(data)
	}
	return
}

// NewLine creates a new Line reader object
func NewLine(input io.Reader, codec encoding.Encoding, bufferSize int) (*Line, error) {

	encoder := codec.NewEncoder()

	// Create newline char based on encoding
	nl, _, err := transform.Bytes(encoder, []byte{'\n'})
	if err != nil {
		return nil, err
	}

	return &Line{
		reader:     input,
		codec:      codec,
		bufferSize: bufferSize,
		nl:         nl,
		decoder:    codec.NewDecoder(),
		inBuffer:   streambuf.New(nil),
		outBuffer:  streambuf.New(nil),
	}, nil
}

// read reader content to string, using charset specified
func readToStringWithCharset(reader io.Reader, charset string) (string, error) {
	charset = strings.ToUpper(charset)
	var data []byte
	var err error
	if charset == "UTF-8" || charset == "UTF8" {
		data, err = ioutil.ReadAll(reader)
	} else {
		if charset == "GBK" || charset == "GB2312" {
			charset = "GB18030"
		}
		var encoder encoding.Encoding
		encoder, err = htmlindex.Get(charset)
		if err != nil {
			return "", err
		}
		data, err = ioutil.ReadAll(transform.NewReader(reader, encoder.NewDecoder()))
	}
	if err != nil {
		return "", err
	}
	return string(data), err
}

func (client *Client) FileInfo(url string, encoding encoding.Encoding) (filename string, size int, status int, errMsg error) {
	status = http.StatusOK

	resp, err := client.Head(url)
	if err != nil {
		status = http.StatusBadGateway
		errMsg = fmt.Errorf("Failed to HEAD from %s: %s", url, err)
		return
	}

	// Filename.
	disposition := resp.Header.Get("Content-Disposition")
	disposition, err = encoding.NewDecoder().String(disposition)
	if err != nil {
		status = http.StatusInternalServerError
		errMsg = fmt.Errorf("Failed to decode Content-Disposition: %s", err)
		return
	}

	// Parse disposition header.
	_, params, err := mime.ParseMediaType(disposition)
	if err != nil {
		status = http.StatusInternalServerError
		errMsg = fmt.Errorf("Failed to parse header Content-Disposition of file at %s: %s", url, err)
		return
	}
	filename = params["filename"]

	// File size.
	sizeStr := resp.Header.Get("Content-Length")
	size, err = strconv.Atoi(sizeStr)
	if err != nil {
		status = http.StatusInternalServerError
		errMsg = fmt.Errorf("Failed to convert Content-Length (%s) to int: %s", sizeStr, err)
		return
	}

	return
}

func ConvToUTF8(b []byte, e encoding.Encoding) (result []byte, err error) {
	reader := transform.NewReader(bytes.NewReader(b), unicode.BOMOverride(e.NewDecoder()))
	return ioutil.ReadAll(reader)
}

func enc(encoding encoding.Encoding) Decoder {
	return trans(encoding.NewDecoder())
}

func dec(e encoding.Encoding) (dir string, t transform.Transformer, err error) {
	return "Decode", e.NewDecoder(), nil
}