Golang utf8.ValidString函数代码示例

// CopyObject copies the source object to the destination.
// The copied object's attributes are overwritten by attrs if non-nil.
func (c *Client) CopyObject(ctx context.Context, srcBucket, srcName string, destBucket, destName string, attrs *ObjectAttrs) (*ObjectAttrs, error) {
	if srcBucket == "" || destBucket == "" {
		return nil, errors.New("storage: srcBucket and destBucket must both be non-empty")
	}
	if srcName == "" || destName == "" {
		return nil, errors.New("storage: srcName and destName must be non-empty")
	}
	if !utf8.ValidString(srcName) {
		return nil, fmt.Errorf("storage: srcName %q is not valid UTF-8", srcName)
	}
	if !utf8.ValidString(destName) {
		return nil, fmt.Errorf("storage: destName %q is not valid UTF-8", destName)
	}
	var rawObject *raw.Object
	if attrs != nil {
		attrs.Name = destName
		if attrs.ContentType == "" {
			return nil, errors.New("storage: attrs.ContentType must be non-empty")
		}
		rawObject = attrs.toRawObject(destBucket)
	}
	o, err := c.raw.Objects.Copy(
		srcBucket, srcName, destBucket, destName, rawObject).Projection("full").Context(ctx).Do()
	if err != nil {
		return nil, err
	}
	return newObject(o), nil
}

func main() {
	s := "Hello, \x90\xA2\x8A\x45" // CP932 encoded version of "Hello, 世界" ， 这里的 s 是string类型，说明string没有字符集的概念

	r, _ := charset.NewReader("CP932", strings.NewReader(s)) // convert from CP932 to UTF-8
	s2_, _ := ioutil.ReadAll(r)
	s2 := string(s2_)
	fmt.Println(s2)                         // => Hello, 世界
	fmt.Println(len(s2))                    // => 13
	fmt.Println(utf8.RuneCountInString(s2)) // => 9
	fmt.Println(utf8.ValidString(s2))       // => true
	fmt.Println(utf8.ValidString(s))        // => false
	fmt.Printf("%T|%#v\n", s, s)            // 注意 %v 与 %#v 的区别

	ss := "This is not utf-8 string \xa1"
	fmt.Println(utf8.ValidString(ss)) // => false

	pice := []int32{20, 30, 40, 90}
	sss := string(pice)                                             // string 似乎执行了内存拷贝，但是不会涉及到字符集的处理（转换或校验）
	fmt.Printf("%T:%p %T:%p:%d\n", pice, pice, sss, &sss, len(sss)) // 为什么打印字符串变量的地址还需要取地址符

	tr, err := charset.TranslatorTo("windows-1252") //需要检查字符集列表
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	_, gbk, err2 := tr.Translate([]byte("utf-8汉字"), true)
	if err2 != nil {
		fmt.Println(err2)
		os.Exit(1)
	}
	fmt.Println(gbk)
}

// CopyTo copies the object to the given dst.
// The copied object's attributes are overwritten by attrs if non-nil.
func (o *ObjectHandle) CopyTo(ctx context.Context, dst *ObjectHandle, attrs *ObjectAttrs) (*ObjectAttrs, error) {
	// TODO(djd): move bucket/object name validation to a single helper func.
	if o.bucket == "" || dst.bucket == "" {
		return nil, errors.New("storage: the source and destination bucket names must both be non-empty")
	}
	if o.object == "" || dst.object == "" {
		return nil, errors.New("storage: the source and destination object names must both be non-empty")
	}
	if !utf8.ValidString(o.object) {
		return nil, fmt.Errorf("storage: object name %q is not valid UTF-8", o.object)
	}
	if !utf8.ValidString(dst.object) {
		return nil, fmt.Errorf("storage: dst name %q is not valid UTF-8", dst.object)
	}
	var rawObject *raw.Object
	if attrs != nil {
		attrs.Name = dst.object
		if attrs.ContentType == "" {
			return nil, errors.New("storage: attrs.ContentType must be non-empty")
		}
		rawObject = attrs.toRawObject(dst.bucket)
	}
	call := o.c.raw.Objects.Copy(o.bucket, o.object, dst.bucket, dst.object, rawObject).Projection("full").Context(ctx)
	if err := applyConds("CopyTo destination", dst.conds, call); err != nil {
		return nil, err
	}
	if err := applyConds("CopyTo source", toSourceConds(o.conds), call); err != nil {
		return nil, err
	}
	obj, err := call.Do()
	if err != nil {
		return nil, err
	}
	return newObject(obj), nil
}

// validateValues reads the values provided as args and validates that they are
// valid UTF-8.
func (c *configCommand) validateValues(ctx *cmd.Context) (map[string]string, error) {
	settings := map[string]string{}
	for k, v := range c.values {
		//empty string is also valid as a setting value
		if v == "" {
			settings[k] = v
			continue
		}

		if v[0] != '@' {
			if !utf8.ValidString(v) {
				return nil, errors.Errorf("value for option %q contains non-UTF-8 sequences", k)
			}
			settings[k] = v
			continue
		}
		nv, err := readValue(ctx, v[1:])
		if err != nil {
			return nil, err
		}
		if !utf8.ValidString(nv) {
			return nil, errors.Errorf("value for option %q contains non-UTF-8 sequences", k)
		}
		settings[k] = nv
	}
	return settings, nil

}

// Run updates the configuration of a service.
func (c *SetCommand) Run(ctx *cmd.Context) error {
	api, err := c.getAPI()
	if err != nil {
		return err
	}
	defer api.Close()

	if c.SettingsYAML.Path != "" {
		b, err := c.SettingsYAML.Read(ctx)
		if err != nil {
			return err
		}
		return block.ProcessBlockedError(api.ServiceSetYAML(c.ServiceName, string(b)), block.BlockChange)
	} else if len(c.SettingsStrings) == 0 {
		return nil
	}
	settings := map[string]string{}
	for k, v := range c.SettingsStrings {
		//empty string is also valid as a setting value
		if v == "" {
			settings[k] = v
			continue
		}

		if v[0] != '@' {
			if !utf8.ValidString(v) {
				return fmt.Errorf("value for option %q contains non-UTF-8 sequences", k)
			}
			settings[k] = v
			continue
		}
		nv, err := readValue(ctx, v[1:])
		if err != nil {
			return err
		}
		if !utf8.ValidString(nv) {
			return fmt.Errorf("value for option %q contains non-UTF-8 sequences", k)
		}
		settings[k] = nv
	}

	result, err := api.ServiceGet(c.ServiceName)
	if err != nil {
		return err
	}

	for k, v := range settings {
		configValue := result.Config[k]

		configValueMap, ok := configValue.(map[string]interface{})
		if ok {
			// convert the value to string and compare
			if fmt.Sprintf("%v", configValueMap["value"]) == v {
				logger.Warningf("the configuration setting %q already has the value %q", k, v)
			}
		}
	}

	return block.ProcessBlockedError(api.ServiceSet(c.ServiceName, settings), block.BlockChange)
}

// fix line endings
func (step *ProblemStep) Normalize(n int64) error {
	step.Step = n
	step.Note = strings.TrimSpace(step.Note)
	if step.Note == "" {
		return fmt.Errorf("missing note for step %d", n+1)
	}
	instructions, err := step.BuildInstructions()
	if err != nil {
		return fmt.Errorf("error building instructions for step %d: %v", n+1, err)
	}
	step.Instructions = instructions
	if step.Weight <= 0.0 {
		// default to 1.0
		step.Weight = 1.0
	}
	clean := make(map[string]string)
	for name, contents := range step.Files {
		parts := strings.Split(name, "/")
		fixed := contents
		if (len(parts) < 2 || !ProblemStepDirectoryWhitelist[parts[0]]) && utf8.ValidString(contents) {
			fixed = fixLineEndings(contents)
			if fixed != contents {
				log.Printf("fixed line endings for %s", name)
			}
		} else if utf8.ValidString(contents) {
			fixed = fixNewLines(contents)
			if fixed != contents {
				log.Printf("fixed newlines for %s", name)
			}
		}
		clean[name] = fixed
	}
	step.Files = clean
	return nil
}

// decodeHeader decodes header, detecting its charset.
// It guarantees to produce utf-8 string or error.
func decodeHeader(rawheader string) (string, error) {
	dec := &mime.WordDecoder{
		CharsetReader: charset.NewReaderLabel,
	}

	header, err := dec.DecodeHeader(rawheader)
	if err != nil {
		return header, err
	}

	if !utf8.ValidString(header) {
		nheader, err := decodeCharset(header, "")
		if err != nil {
			return header, err
		}

		if !utf8.ValidString(nheader) {
			return header, fmt.Errorf("decode header: non-utf8 byte left after decode")
		}

		return nheader, nil
	}

	return header, nil
}

func ExampleValidString() {
	valid := "Hello, 世界"
	invalid := string([]byte{0xff, 0xfe, 0xfd})

	fmt.Println(utf8.ValidString(valid))
	fmt.Println(utf8.ValidString(invalid))
	// Output:
	// true
	// false
}

func (s *DoSuite) SetUpTest(c *gc.C) {
	s.BaseActionSuite.SetUpTest(c)
	s.dir = c.MkDir()
	c.Assert(utf8.ValidString(validParamsYaml), jc.IsTrue)
	c.Assert(utf8.ValidString(invalidParamsYaml), jc.IsTrue)
	c.Assert(utf8.ValidString(invalidUTFYaml), jc.IsFalse)
	setupValueFile(c, s.dir, "validParams.yml", validParamsYaml)
	setupValueFile(c, s.dir, "invalidParams.yml", invalidParamsYaml)
	setupValueFile(c, s.dir, "invalidUTF.yml", invalidUTFYaml)
}

func (s *SetSuite) SetUpTest(c *gc.C) {
	s.FakeJujuHomeSuite.SetUpTest(c)
	s.fakeServiceAPI = &fakeServiceAPI{serviceName: "dummy-service"}

	s.dir = c.MkDir()
	c.Assert(utf8.ValidString(validSetTestValue), jc.IsTrue)
	c.Assert(utf8.ValidString(invalidSetTestValue), jc.IsFalse)
	setupValueFile(c, s.dir, "valid.txt", validSetTestValue)
	setupValueFile(c, s.dir, "invalid.txt", invalidSetTestValue)
	setupBigFile(c, s.dir)
	setupConfigFile(c, s.dir)
}

func (s *SetSuite) SetUpTest(c *gc.C) {
	s.JujuConnSuite.SetUpTest(c)
	ch := s.AddTestingCharm(c, "dummy")
	svc := s.AddTestingService(c, "dummy-service", ch)
	s.svc = svc
	s.dir = c.MkDir()
	c.Assert(utf8.ValidString(validSetTestValue), gc.Equals, true)
	c.Assert(utf8.ValidString(invalidSetTestValue), gc.Equals, false)
	setupValueFile(c, s.dir, "valid.txt", validSetTestValue)
	setupValueFile(c, s.dir, "invalid.txt", invalidSetTestValue)
	setupBigFile(c, s.dir)
	setupConfigFile(c, s.dir)
}

func newEntry(path, prefix string, fileInfo os.FileInfo) (*entry, error) {
	if !utf8.ValidString(path) || !utf8.ValidString(prefix) || !utf8.ValidString(fileInfo.Name()) {
		return nil, errors.New("pathname is not valid UTF8")
	}

	entry := entry{
		path:     path,
		prefix:   prefix,
		fileInfo: fileInfo,
	}

	return &entry, nil
}

func logMorph(log irclogsme.LogMessage) Log {
	res := Log{
		Id:    log.Id.String(),
		Time:  log.Time,
		Nick:  log.Nick,
		Ident: log.Ident,
		Host:  log.Host,
	}
	// now to specify
	switch log.Type {
	case irclogsme.LMT_PRIVMSG:
		res.Type = "privmsg"
		res.Data = log.Payload
	case irclogsme.LMT_NOTICE:
		res.Type = "notice"
		res.Data = log.Payload
	case irclogsme.LMT_JOIN:
		res.Type = "join"
	case irclogsme.LMT_PART:
		res.Type = "part"
		res.Data = log.Payload
	case irclogsme.LMT_TOPIC:
		res.Type = "topic"
		res.Data = log.Payload
	case irclogsme.LMT_QUIT:
		res.Type = "quit"
		res.Data = log.Payload
	case irclogsme.LMT_ACTION:
		res.Type = "action"
		res.Data = log.Payload
	case irclogsme.LMT_KICK:
		res.Type = "kick"
		lk := LogKick{Target: log.Target.(string)}
		if message, ok := log.Payload.(string); ok {
			lk.Message = message
		}
		res.Data = lk
	}
	if sdata, ok := res.Data.(string); ok {
		if !utf8.ValidString(sdata) {
			res.Data = "[invalid unicode]"
		}
	} else if sdata, ok := res.Data.(LogKick); ok {
		if !utf8.ValidString(sdata.Message) {
			lk := res.Data.(LogKick)
			lk.Message = "[invalid unicode]"
			res.Data = lk
		}
	}
	return res
}

func fixDisplayNameForAppGroup(info ItemInfo) ItemInfo {
	if info.FileType == uint16(gio.FileTypeDirectory) && strings.HasPrefix(info.BaseName, AppGroupPrefix) {
		info.DisplayName = strings.TrimPrefix(info.DisplayName, AppGroupPrefix)
	}

	// fix bug: name contains invalid coding
	if !utf8.ValidString(info.DisplayName) {
		info.DisplayName = fmt.Sprintf("%q", info.DisplayName)
	}
	if !utf8.ValidString(info.BaseName) {
		info.BaseName = fmt.Sprintf("%q", info.BaseName)
	}

	return info
}

func findUnmanagedFiles(dir string, rpmFiles map[string]string, rpmDirs map[string]bool,
	unmanagedFiles map[string]string, ignoreList map[string]bool) {
	files, _ := readDir(dir)
	for _, f := range files {
		fileName := dir + f.Name()
		if !utf8.ValidString(fileName) {
			fmt.Fprintln(os.Stderr, fileName, "contains invalid UTF-8 characters. Skipping.")
		} else {
			if _, ok := ignoreList[fileName]; !ok {
				if f.IsDir() {
					if _, ok := rpmDirs[fileName]; ok {
						findUnmanagedFiles(fileName+"/", rpmFiles, rpmDirs, unmanagedFiles, ignoreList)
					} else {
						if !hasManagedDirs(fileName, rpmDirs) {
							unmanagedFiles[fileName+"/"] = "dir"
						}
					}
				} else {
					if _, ok := rpmFiles[fileName]; !ok {
						if f.Mode()&
							(os.ModeSocket|os.ModeNamedPipe|os.ModeDevice|os.ModeCharDevice) != 0 {
							// Ignore sockets, named pipes and devices
						} else if f.Mode()&os.ModeSymlink == os.ModeSymlink {
							unmanagedFiles[fileName] = "link"
						} else {
							unmanagedFiles[fileName] = "file"
						}
					}
				}
			}
		}
	}
}