Golang utf8.Valid函数代码示例

func ExampleValid() {
	valid := []byte("Hello, 世界")
	invalid := []byte{0xff, 0xfe, 0xfd}

	fmt.Println(utf8.Valid(valid))
	fmt.Println(utf8.Valid(invalid))
	// Output:
	// true
	// false
}

// IsUTF8 reports whether the blob is entirely UTF-8.
func (b *Blob) IsUTF8() bool {
	if b.mem != nil {
		return utf8.Valid(b.mem)
	}
	rc := b.Open()
	defer rc.Close()
	slurp, err := ioutil.ReadAll(rc)
	if err != nil {
		return false
	}
	return utf8.Valid(slurp)
}

func makestatic() error {
	f, err := os.Create("static.go")
	if err != nil {
		return err
	}
	defer f.Close()
	w := bufio.NewWriter(f)
	fmt.Fprintf(w, "%v\npackage static\n\n", warning)
	fmt.Fprintf(w, "var Files = map[string]string{\n")
	for _, fn := range files {
		b, err := ioutil.ReadFile(fn)
		if err != nil {
			return err
		}
		fmt.Fprintf(w, "\t%q: ", fn)
		if utf8.Valid(b) {
			fmt.Fprintf(w, "`%s`", sanitize(b))
		} else {
			fmt.Fprintf(w, "%q", b)
		}
		fmt.Fprintln(w, ",\n")
	}
	fmt.Fprintln(w, "}")
	if err := w.Flush(); err != nil {
		return err
	}
	return f.Close()
}

func backup(c *cli.Context) (err error) {
	// Get KV client
	client, backupResult, err := getConnectionFromFlags(c)
	if err != nil {
		return
	}
	kv := client.KV()

	// Dump all
	pairs, _, err := kv.List(c.GlobalString("prefix"), &api.QueryOptions{})
	if err != nil {
		return
	}
	bkup := map[string]valueEnc{}
	for _, p := range pairs {
		validUtf8 := utf8.Valid(p.Value)
		if validUtf8 {
			bkup[p.Key] = valueEnc{"", string(p.Value)}
		} else {
			sEnc := base64.StdEncoding.EncodeToString(p.Value)
			bkup[p.Key] = valueEnc{"base64", sEnc}
		}
	}
	backupResult.Values = bkup

	// Send results to outfile (if defined) or stdout
	dumpOutput(c.String("outfile"), backupResult)

	return
}

func makestatic() error {
	f, err := os.Create("static.go")
	if err != nil {
		return err
	}
	defer f.Close()
	buf := new(bytes.Buffer)
	fmt.Fprintf(buf, "%v\npackage static\n\n", warning)
	fmt.Fprintf(buf, "var Files = map[string]string{\n")
	for _, fn := range files {
		b, err := ioutil.ReadFile(fn)
		if err != nil {
			return err
		}
		fmt.Fprintf(buf, "\t%q: ", fn)
		if utf8.Valid(b) {
			fmt.Fprintf(buf, "`%s`", sanitize(b))
		} else {
			fmt.Fprintf(buf, "%q", b)
		}
		fmt.Fprintln(buf, ",\n")
	}
	fmt.Fprintln(buf, "}")
	fmtbuf, err := format.Source(buf.Bytes())
	if err != nil {
		return err
	}
	return ioutil.WriteFile("static.go", fmtbuf, 0666)
}

// DecryptTags looks for any tagged data of the form [gosecret|authtext|ciphertext|initvector|keyname] in the
// input content byte array and replaces each with a decrypted version of the ciphertext.  Note that the
// input content must be valid UTF-8.  The second parameter is the path to the directory in which keyfiles
// live.  For each |keyname| in a gosecret block, there must be a corresponding file of the same name in the
// keystore directory.
// DecryptTags returns a []byte with all [gosecret] blocks replaced by plaintext.
func DecryptTags(content []byte, keyroot string) ([]byte, error) {

	if !utf8.Valid(content) {
		return nil, errors.New("File is not valid UTF-8")
	}

	content = gosecretRegex.ReplaceAllFunc(content, func(match []byte) []byte {
		matchString := string(match)
		matchString = matchString[:len(matchString)-1]
		parts := strings.Split(matchString, "|")

		if len(parts) < 5 {
			// Block is not encrypted.  Noop.
			return match
		} else {
			plaintext, err := decryptTag(parts, keyroot)
			if err != nil {
				fmt.Println("Unable to decrypt tag", err)
				return nil
			}

			return plaintext
		}
	})

	return content, nil
}

func (e *echoHandler) extractBody(r io.ReadCloser, fileName, mimeType string) (body, error) {
	defer r.Close()
	bytes, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}
	//either display it as a string or store as file
	if utf8.Valid(bytes) {
		return body(string(bytes)), nil
	}

	//calc mime
	if (mimeType == "" || mimeType == defaultmtype) && fileName != "" {
		mimeType = mime.TypeByExtension(filepath.Ext(fileName))
	}
	//hash bytes
	hash := md5.New()
	hash.Write([]byte(mimeType + "|"))
	hash.Write(bytes)
	md5 := hex.EncodeToString(hash.Sum(nil))

	b := &bodyValues{
		Length: len(bytes),
		Type:   mimeType,
		MD5:    md5,
		URL:    "/file/" + md5,
	}
	e.cache.Add(b.MD5, fileName, mimeType, bytes)
	return b, nil
}

// Copy everything from the pty master to the socket.
func handleOutputSock(ptym *os.File, conn *net.TCPConn) {
	buf := make([]byte, 512)
	var payload, overflow []byte
	// TODO: more graceful exit on socket close / process exit.
	for {
		n, err := ptym.Read(buf)
		if err != nil {
			fmt.Println("failed to read from pty master: ", err)
			return
		}

		// Empty the overflow from the last read into the payload first.
		payload = append(payload[0:], overflow...)
		overflow = nil
		// Then empty the new buf read into the payload.
		payload = append(payload, buf[:n]...)

		// Strip out any incomplete utf-8 from current payload into overflow.
		for !utf8.Valid(payload) {
			overflow = append(overflow[:0], append(payload[len(payload)-1:], overflow[0:]...)...)
			payload = payload[:len(payload)-1]
		}

		// Send out the finished payload as long as it's not empty.
		if len(payload) >= 1 {
			_, err = conn.Write(payload)
			if err != nil {
				fmt.Println("Write: ", err)
			}
		}

		// Empty the payload.
		payload = nil
	}
}

/**
Encode string to CBOR binary string
*/
func (encoder *cborEncode) encodeString(variable string) (bool, error) {
	byteBuf := []byte(variable)

	majorType := majorTypeUtf8String

	if !utf8.Valid(byteBuf) {
		majorType = majorTypeByteString
	}

	initByte, err := packNumber(majorType, uint64(len(byteBuf)))

	if err != nil {
		return false, err
	}

	_, err = encoder.buff.Write(initByte)

	if err != nil {
		return false, err
	}

	_, err = encoder.buff.Write(byteBuf)

	if err != nil {
		return false, err
	}

	return true, nil
}

func script(name string) {
	file, err := os.Open(name)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	bytes, err := ioutil.ReadAll(file)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	if !utf8.Valid(bytes) {
		fmt.Fprintf(os.Stderr, "source %v is not valid UTF-8\n", name)
		os.Exit(1)
	}
	src := string(bytes)

	ev := eval.NewEvaluator()

	n, pe := parse.Parse(name, src)
	if pe != nil {
		fmt.Print(pe.(*util.ContextualError).Pprint())
		os.Exit(1)
	}

	ee := ev.Eval(name, src, n)
	if ee != nil {
		fmt.Print(ee.(*util.ContextualError).Pprint())
		os.Exit(1)
	}
}

func main() {
	var b1 []byte = []byte("안녕하세요")
	fmt.Println(utf8.Valid(b1)) // true: "안녕하세요"는 UTF-8이 맞으므로 true
	var b2 []byte = []byte{0xff, 0xf1, 0xc1}
	fmt.Println(utf8.Valid(b2)) // false: 0xff 0xf1 0xc1은 UTF-8이 아니므로 false

	var r1 rune = '한'
	fmt.Println(utf8.ValidRune(r1)) // true: '한'은 UTF-8이 맞으므로 true
	var r2 rune = 0x11111111
	fmt.Println(utf8.ValidRune(r2)) // false: 0x11111111은 UTF-8이 아니므로 false

	var s1 string = "한글"
	fmt.Println(utf8.ValidString(s1)) // true: "한글"은 UTF-8이 맞으므로 true
	var s2 string = string([]byte{0xff, 0xf1, 0xc1})
	fmt.Println(utf8.ValidString(s2)) // false: 0xff 0xf1 0xc1은 UTF-8이 아니므로 false
}

// ReadHistory reads scrollback history from r. Returns the number of lines
// read, and any read error (except io.EOF).
func (s *State) ReadHistory(r io.Reader) (num int, err error) {
	s.historyMutex.Lock()
	defer s.historyMutex.Unlock()

	in := bufio.NewReader(r)
	num = 0
	for {
		line, part, err := in.ReadLine()
		if err == io.EOF {
			break
		}
		if err != nil {
			return num, err
		}
		if part {
			return num, fmt.Errorf("line %d is too long", num+1)
		}
		if !utf8.Valid(line) {
			return num, fmt.Errorf("invalid string at line %d", num+1)
		}
		num++
		s.history = append(s.history, string(line))
		if len(s.history) > HistoryLimit {
			s.history = s.history[1:]
		}
	}
	return num, nil
}

func uncompressed_memcopy(w io.Writer, asset *Asset, r io.Reader) error {
	_, err := fmt.Fprintf(w, `var _%s = []byte(`, asset.Func)
	if err != nil {
		return err
	}

	b, err := ioutil.ReadAll(r)
	if err != nil {
		return err
	}
	if utf8.Valid(b) {
		fmt.Fprintf(w, "`%s`", sanitize(b))
	} else {
		fmt.Fprintf(w, "%q", b)
	}

	_, err = fmt.Fprintf(w, `)

func %s_bytes() ([]byte, error) {
	return _%s, nil
}

`, asset.Func, asset.Func)
	return err
}

// ReadHistory reads scrollback history from r. Returns the number of lines
// read, and any read error (except io.EOF).
func (s *State) ReadHistory(r io.Reader) (num int, err error) {
	in := bufio.NewReader(r)
	num = 0
	for {
		line, part, err := in.ReadLine()
		if err == io.EOF {
			break
		}
		if err != nil {
			return num, err
		}
		if part {
			return num, errors.New("Line too long")
		}
		if !utf8.Valid(line) {
			return num, errors.New("Invalid string")
		}
		num++
		s.history = append(s.history, string(line))
		if len(s.history) > HistoryLimit {
			s.history = s.history[1:]
		}
	}
	return num, nil
}

func parseHeaders(head []byte) (map[string]interface{}, error) {
	if !utf8.Valid(head) {
		return nil, fmt.Errorf("header is not utf8")
	}
	headers := make(map[string]interface{})
	lines := strings.Split(string(head), "\n")
	for i := 0; i < len(lines); {
		entry := lines[i]
		nameValueSplit := strings.Index(entry, ":")
		if nameValueSplit == -1 {
			return nil, fmt.Errorf("header entry missing ':' separator: %q", entry)
		}
		name := entry[:nameValueSplit]
		if !headerNameSanity.MatchString(name) {
			return nil, fmt.Errorf("invalid header name: %q", name)
		}

		consumed := nameValueSplit + 1
		var value interface{}
		var err error
		value, i, err = parseEntry(consumed, i, lines, 0)
		if err != nil {
			return nil, err
		}

		if _, ok := headers[name]; ok {
			return nil, fmt.Errorf("repeated header: %q", name)
		}

		headers[name] = value
	}
	return headers, nil
}