Golang transform.NewWriter函数代码示例

//「Golangで文字コード判定」qiita.com/nobuhito/items/ff782f64e32f7ed95e43
func transEnc(text string, encode string) (string, error) {
	body := []byte(text)
	var f []byte

	encodings := []string{"sjis", "utf-8"}
	if encode != "" {
		encodings = append([]string{encode}, encodings...)
	}
	for _, enc := range encodings {
		if enc != "" {
			ee, _ := charset.Lookup(enc)
			if ee == nil {
				continue
			}
			var buf bytes.Buffer
			ic := transform.NewWriter(&buf, ee.NewDecoder())
			_, err := ic.Write(body)
			if err != nil {
				continue
			}
			err = ic.Close()
			if err != nil {
				continue
			}
			f = buf.Bytes()
			break
		}
	}
	return string(f), nil
}

func TestReaderEncodings(t *testing.T) {
	for _, test := range tests {
		t.Logf("test codec: %v", test.encoding)

		codec, ok := findEncoding(test.encoding)
		if !ok {
			t.Errorf("can not find encoding '%v'", test.encoding)
			continue
		}

		buffer := bytes.NewBuffer(nil)

		// write with encoding to buffer
		writer := transform.NewWriter(buffer, codec.NewEncoder())
		var expectedCount []int
		for _, line := range test.strings {
			writer.Write([]byte(line))
			writer.Write([]byte{'\n'})
			expectedCount = append(expectedCount, buffer.Len())
		}

		// create line reader
		reader, err := newLineReader(buffer, codec, 1024)
		if err != nil {
			t.Errorf("failed to initialize reader: %v", err)
			continue
		}

		// read decodec lines from buffer
		var readLines []string
		var byteCounts []int
		current := 0
		for {
			bytes, sz, err := reader.next()
			if sz > 0 {
				readLines = append(readLines, string(bytes[:len(bytes)-1]))
			}

			if err != nil {
				break
			}

			current += sz
			byteCounts = append(byteCounts, current)
		}

		// validate lines and byte offsets
		if len(test.strings) != len(readLines) {
			t.Errorf("number of lines mismatch (expected=%v actual=%v)",
				len(test.strings), len(readLines))
			continue
		}
		for i := range test.strings {
			expected := test.strings[i]
			actual := readLines[i]
			assert.Equal(t, expected, actual)
			assert.Equal(t, expectedCount[i], byteCounts[i])
		}
	}
}

// Shift-JIS -> UTF-8
func to_utf8(str string) (string, error) {
	body, err := ioutil.ReadAll(transform.NewReader(strings.NewReader(str), japanese.ShiftJIS.NewEncoder()))
	if err != nil {
		return "", err
	}

	var f []byte
	encodings := []string{"sjis", "utf-8"}
	for _, enc := range encodings {
		if enc != "" {
			ee, _ := charset.Lookup(enc)
			if ee == nil {
				continue
			}
			var buf bytes.Buffer
			ic := transform.NewWriter(&buf, ee.NewDecoder())
			_, err := ic.Write(body)
			if err != nil {
				continue
			}
			err = ic.Close()
			if err != nil {
				continue
			}
			f = buf.Bytes()
			break
		}
	}
	return string(f), nil
}

func getWriter(writer io.Writer) io.Writer {
	encoding := getEncoding()
	if encoding == nil {
		return writer
	}
	return transform.NewWriter(writer, encoding.NewEncoder())
}

func TestReaderPartialWithEncodings(t *testing.T) {
	for _, test := range tests {
		t.Logf("test codec: %v", test.encoding)

		codecFactory, ok := encoding.FindEncoding(test.encoding)
		if !ok {
			t.Errorf("can not find encoding '%v'", test.encoding)
			continue
		}

		buffer := bytes.NewBuffer(nil)
		codec, _ := codecFactory(buffer)

		writer := transform.NewWriter(buffer, codec.NewEncoder())
		reader, err := newLineReader(buffer, codec, 1024)
		if err != nil {
			t.Errorf("failed to initialize reader: %v", err)
			continue
		}

		var expected []string
		var partials []string
		lastString := ""
		for _, str := range test.strings {
			writer.Write([]byte(str))
			lastString += str
			expected = append(expected, lastString)

			line, sz, err := reader.next()
			assert.NotNil(t, err)
			assert.Equal(t, 0, sz)
			assert.Nil(t, line)

			partial, _, err := reader.partial()
			partials = append(partials, string(partial))
			t.Logf("partials: %v", partials)
		}

		// finish line:
		writer.Write([]byte{'\n'})

		// finally read line
		line, _, err := reader.next()
		assert.Nil(t, err)
		t.Logf("line: '%v'", line)

		// validate partial lines
		if len(test.strings) != len(expected) {
			t.Errorf("number of lines mismatch (expected=%v actual=%v)",
				len(test.strings), len(partials))
			continue
		}
		for i := range expected {
			assert.Equal(t, expected[i], partials[i])
		}

		assert.Equal(t, lastString+"\n", string(line))
	}
}

func newWriter(option *Option) io.Writer {
	encoder := func() io.Writer {
		switch option.OutputEncode {
		case "sjis":
			return transform.NewWriter(os.Stdout, japanese.ShiftJIS.NewEncoder())
		case "euc":
			return transform.NewWriter(os.Stdout, japanese.EUCJP.NewEncoder())
		case "jis":
			return transform.NewWriter(os.Stdout, japanese.ISO2022JP.NewEncoder())
		default:
			return os.Stdout
		}
	}()
	if option.EnableColor {
		return ansicolor.NewAnsiColorWriter(encoder)
	}
	return encoder
}

func ConvTo(b []byte, e encoding.Encoding) (result []byte, err error) {
	w := new(bytes.Buffer)
	writer := transform.NewWriter(w, e.NewEncoder())
	defer writer.Close()

	if _, err = writer.Write(b); err != nil {
		return
	}
	return w.Bytes(), nil
}

func (r *ResultCSV) Do(ctx WebContext) error {
	w := ctx.ResponseWriter()
	w.Header().Set("Content-Type", "text/csv; char=utf-8")
	if r.Attachment {
		w.Header().Set("Content-Disposition", "attachment")
	}
	w.WriteHeader(r.Code)
	csvw := csv.NewWriter(transform.NewWriter(w, japanese.ShiftJIS.NewEncoder()))
	return csvw.WriteAll(r.Data)
}

func decodeBy(charset string, dst io.Writer) io.Writer {
	if "UTF-8" == strings.ToUpper(charset) || "UTF8" == strings.ToUpper(charset) {
		return dst
	}
	cs := GetCharset(charset)
	if nil == cs {
		panic("charset '" + charset + "' is not exists.")
	}

	return transform.NewWriter(dst, cs.NewDecoder())
}

func EncodeReader(s io.Reader, enc string) ([]byte, error) {
	e, _ := charset.Lookup(enc)
	if e == nil {
		return nil, errors.New(fmt.Sprintf("unsupported charset: %q", enc))
	}
	var buf bytes.Buffer
	writer := transform.NewWriter(&buf, e.NewEncoder())
	_, err := io.Copy(writer, s)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// Convert unicode text to ASCII text
// using specific codepage mapping.
func convertUnicodeToAscii(text string,
	codepage encoding.Encoding) []byte {
	b := []byte(text)
	// fmt.Printf("Text length: %d\n", len(b))
	var buf bytes.Buffer
	if codepage == nil {
		codepage = charmap.Windows1252
	}
	w := transform.NewWriter(&buf, codepage.NewEncoder())
	defer w.Close()
	w.Write(b)
	// fmt.Printf("Buffer length: %d\n", len(buf.Bytes()))
	return buf.Bytes()
}

func saveDatasetAsCsv(dataset []Data, filePath string) {
	f, err := os.Create(filePath)
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	var w *csv.Writer
	if runtime.GOOS == "windows" {
		// on Windows, use Shift-JIS to open csv file via Microsoft Excel.
		converter := bufio.NewWriter(transform.NewWriter(f, japanese.ShiftJIS.NewEncoder()))
		w = csv.NewWriter(converter)
	} else {
		w = csv.NewWriter(f)
	}
	defer w.Flush()

	// Write header first
	header := []string{
		// "日付",
		"証券会社名",
		"n225_sell",
		"n225_buy",
		"n225_net",
		"topix_sell",
		"topix_buy",
		"topix_net",
		"net_total",
	}
	w.Write(header)

	// Write dataset
	for _, data := range dataset {
		var record []string
		// record = append(record, obj.InfoDate)
		record = append(record, data.Company)
		record = append(record, data.N225Sell)
		record = append(record, data.N225Buy)
		record = append(record, data.N225Net)
		record = append(record, data.TopixSell)
		record = append(record, data.TopixBuy)
		record = append(record, data.TopixNet)
		record = append(record, data.NetTotal)
		w.Write(record)
	}
}

//Conversion
func Conversion(inStream io.Reader, outStream io.Writer) error {
	//read from stream (Shift-JIS to UTF-8)
	scanner := bufio.NewScanner(transform.NewReader(inStream, japanese.ShiftJIS.NewDecoder()))
	list := make([]string, 0)
	for scanner.Scan() {
		list = append(list, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		return err
	}
	//write to stream (UTF-8 to EUC-JP)
	writer := bufio.NewWriter(transform.NewWriter(outStream, japanese.EUCJP.NewEncoder()))
	for _, line := range list {
		if _, err := fmt.Fprintln(writer, line); err != nil {
			return err
		}
	}
	return writer.Flush()
}

func (w *ReportWriter) writeCsv(reports []Report) error {
	wr := w.w
	if w.dialect.Encoding == "sjis" {
		log.Info("use ShiftJIS encoder for output.")
		encoder := japanese.ShiftJIS.NewEncoder()
		wr = transform.NewWriter(wr, encoder)
	}
	writer := csv.NewWriter(wr)
	if w.dialect.Comma != 0 {
		writer.Comma = w.dialect.Comma
	}
	for i, report := range reports {
		if i > 0 {
			writer.Write(nil)
		}
		log.Debugf("[%d] write csv file", i+1)
		w.writeCsvOne(writer, report)
	}
	writer.Flush()
	return writer.Error()
}

func encode(data []byte, charsetName string) ([]byte, error) {
	encoding, _ := charset.Lookup(charsetName)
	if encoding == nil {
		return nil, fmt.Errorf("Unsupported charset: %v", charsetName)
	}

	reader := bytes.NewReader(data)
	var b bytes.Buffer
	writer := bufio.NewWriter(&b)

	encodeWriter := transform.NewWriter(writer, encoding.NewEncoder())
	if _, err := io.Copy(encodeWriter, reader); err != nil {
		return nil, err
	}
	if err := writer.Flush(); err != nil {
		return nil, err
	}

	if isUTF8Charset(charsetName) {
		return addBOM(b.Bytes()), nil
	}
	return b.Bytes(), nil
}