Golang unicode.IsSpace函数代码示例

func (n NeologdNormalizer) EliminateSpace(s string) string {
	var (
		b    bytes.Buffer
		prev rune
	)
	for p := 0; p < len(s); {
		c, w := utf8.DecodeRuneInString(s[p:])
		p += w
		if !unicode.IsSpace(c) {
			b.WriteRune(c)
			prev = c
			continue
		}
		for p < len(s) {
			c0, w0 := utf8.DecodeRuneInString(s[p:])
			p += w0
			if !unicode.IsSpace(c0) {
				if unicode.In(prev, unicode.Latin, latinSymbols) &&
					unicode.In(c0, unicode.Latin, latinSymbols) {
					b.WriteRune(' ')
				}
				b.WriteRune(c0)
				prev = c0
				break
			}
		}

	}
	return b.String()
}

// bufio.Scanner function to split data by words and quoted strings
func scanStrings(data []byte, atEOF bool) (advance int, token []byte, err error) {
	// Skip leading spaces.
	start := 0
	for width := 0; start < len(data); start += width {
		var r rune
		r, width = utf8.DecodeRune(data[start:])
		if !unicode.IsSpace(r) {
			break
		}
	}

	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}

	// Scan until space, marking end of word.
	inquote := false
	for width, i := 0, start; i < len(data); i += width {
		var r rune
		r, width = utf8.DecodeRune(data[i:])
		if r == '"' {
			inquote = !inquote
			continue
		}
		if unicode.IsSpace(r) && !inquote {
			return i + width, data[start:i], nil
		}
	}
	// If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
	if atEOF && len(data) > start {
		return len(data), data[start:], nil
	}
	// Request more data.
	return 0, nil, nil
}

// Test white space table matches the Unicode definition.
func TestSpace(t *testing.T) {
	for r := rune(0); r <= utf8.MaxRune; r++ {
		if IsSpace(r) != unicode.IsSpace(r) {
			t.Fatalf("white space property disagrees: %#U should be %t", r, unicode.IsSpace(r))
		}
	}
}

// top returns offset to start of an match.
func (c *ctx) top(tail int, w string) int {
	for len(w) > 0 {
		if tail <= 0 {
			debug.Printf("over backtrack: w=%q", w)
			return -1
		}
		wr, wn := utf8.DecodeLastRuneInString(w)
		cr, cn := utf8.DecodeLastRuneInString(c.content[:tail])
		tail -= cn
		if unicode.IsSpace(wr) {
			if !unicode.IsSpace(cr) {
				// no spaces which required.
				debug.Printf("not space: tail=%d w=%q cr=%q", tail, w, cr)
				return -1
			}
			w = w[:len(w)-wn]
			continue
		}
		if unicode.IsSpace(cr) {
			continue
		}
		w = w[:len(w)-wn]
		if cr != wr {
			// didn't match runes.
			debug.Printf("not match: tail=%d w=%q cr=%q wr=%q",
				tail, w, cr, wr)
			return -1
		}
	}
	return tail
}

// scanWords is a split function for a Scanner that returns each
// space-separated word of text, with surrounding spaces deleted. It will
// never return an empty string. The definition of space is set by
// unicode.IsSpace.
func scanWords(data []byte, atEOF bool) (advance int, token []byte, err error) {
	// Skip leading spaces.
	start := 0
	for width := 0; start < len(data); start += width {
		var r rune
		r, width = utf8.DecodeRune(data[start:])
		if !unicode.IsSpace(r) {
			break
		}
	}
	quote := false
	// Scan until space, marking end of word.
	for width, i := 0, start; i < len(data); i += width {
		var r rune
		r, width = utf8.DecodeRune(data[i:])
		switch {
		case i == 0 && r == '"':
			quote = true
		case !quote && unicode.IsSpace(r):
			return i + width, data[start:i], nil
		case quote && r == '"':
			return i + width, data[start+width : i], nil
		}
	}
	// If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
	if atEOF && len(data) > start {
		return len(data), data[start:], nil
	}
	// Request more data.
	return start, nil, nil
}

// scanWordsKeepPrefix is a split function for a Scanner that returns each
// space-separated word of text, with prefixing spaces included. It will never
// return an empty string. The definition of space is set by unicode.IsSpace.
//
// Adapted from bufio.ScanWords().
func scanTokensKeepPrefix(data []byte, atEOF bool) (advance int, token []byte, err error) {
	// Skip leading spaces.
	start := 0
	for width := 0; start < len(data); start += width {
		var r rune
		r, width = utf8.DecodeRune(data[start:])
		if !unicode.IsSpace(r) {
			break
		}
	}
	if atEOF && len(data) == 0 || start == len(data) {
		return len(data), data, nil
	}
	if len(data) > start && data[start] == '#' {
		return scanLinesKeepPrefix(data, atEOF)
	}
	// Scan until space, marking end of word.
	for width, i := 0, start; i < len(data); i += width {
		var r rune
		r, width = utf8.DecodeRune(data[i:])
		if unicode.IsSpace(r) {
			return i, data[:i], nil
		}
	}
	// If we're at EOF, we have a final, non-empty, non-terminated word. Return it.
	if atEOF && len(data) > start {
		return len(data), data, nil
	}
	// Request more data.
	return 0, nil, nil
}

//converts a string into a slice of strings.  symbols and contiguous strings of any other type
//are returned as individual elements.  all whitespace is excluded
func getTokens(value string) []string {
	var buffer []rune
	var result []string
	chars := []rune(value)
	for i, r := range chars {
		if !unicode.IsLetter(r) && !unicode.IsNumber(r) && !unicode.IsDigit(r) && !unicode.IsSpace(r) {
			if len(buffer) > 0 {
				result = append(result, string(buffer))
				buffer = nil
			}
			result = append(result, string(r))
		} else if unicode.IsSpace(r) {
			if len(buffer) > 0 {
				result = append(result, string(buffer))
			}
			buffer = nil
		} else {
			buffer = append(buffer, r)
			if i == len(chars)-1 {
				result = append(result, string(buffer))
			}
		}
	}
	return result
}

// Move cursor forward to beginning of the previous word.
// Skips the rest of the current word, if any, unless is located at its
// first character. Returns true if the move was successful, false if EOF reached.
func (c *Cursor) PrevWord() bool {
	isNotSpace := func(r rune) bool {
		return !unicode.IsSpace(r)
	}
	for {
		// Skip space until we find a word character.
		// Re-try if we reached beginning-of-line.
		if !c.PrevRuneFunc(isNotSpace) {
			return false
		}
		if !c.BOL() {
			break
		}
	}
	r, _ := c.RuneBefore()
	if isNotSpace(r) {
		// Lowercase word motion differentiates words consisting of
		// (A-Z0-9_) and any other non-whitespace character. Skip until
		// we find either the other word type or whitespace.
		if utils.IsWord(r) {
			c.PrevRuneFunc(func(r rune) bool {
				return !utils.IsWord(r) || unicode.IsSpace(r)
			})
		} else {
			c.PrevRuneFunc(func(r rune) bool {
				return utils.IsWord(r) || unicode.IsSpace(r)
			})
		}
	}
	return !c.BOL()
}

func splitSections(s string) (sections []string) {
	var i, j int
	var quote bool = false
	var section string

	i = 0
	for i < len(s) {
		section = ""
		for j = i; j < len(s); j++ {
			if s[j] == '\'' {
				quote = !quote
			} else if unicode.IsSpace(rune(s[j])) && !quote {
				break
			} else {
				section = section + string(s[j])
			}
		}

		sections = append(sections, section)

		for i = j; i < len(s); i++ {
			if !unicode.IsSpace(rune(s[i])) {
				break
			}
		}
	}

	return sections
}

func (self *TextPreview) Render(context *Context, writer *utils.XMLWriter) (err error) {
	if len(self.PlainText) < self.MaxLength {
		writer.Content(self.PlainText)
	} else {
		shortLength := self.ShortLength
		if shortLength == 0 {
			shortLength = self.MaxLength
		}

		// If in the middle of a word, go back to space before it
		for shortLength > 0 && !unicode.IsSpace(rune(self.PlainText[shortLength-1])) {
			shortLength--
		}

		// If in the middle of space, go back to word before it
		for shortLength > 0 && unicode.IsSpace(rune(self.PlainText[shortLength-1])) {
			shortLength--
		}

		writer.Content(self.PlainText[:shortLength])
		writer.Content("... ")
		if self.MoreLink != nil {
			writer.OpenTag("a")
			writer.Attrib("href", self.MoreLink.URL(context.PathArgs...))
			writer.AttribIfNotDefault("title", self.MoreLink.LinkTitle(context))
			content := self.MoreLink.LinkContent(context)
			if content != nil {
				err = content.Render(context, writer)
			}
			writer.ForceCloseTag() // a
		}
	}
	return err
}

// The HEALTHCHECK command is like parseMaybeJSON, but has an extra type argument.
func parseHealthConfig(rest string) (*Node, map[string]bool, error) {
	// Find end of first argument
	var sep int
	for ; sep < len(rest); sep++ {
		if unicode.IsSpace(rune(rest[sep])) {
			break
		}
	}
	next := sep
	for ; next < len(rest); next++ {
		if !unicode.IsSpace(rune(rest[next])) {
			break
		}
	}

	if sep == 0 {
		return nil, nil, nil
	}

	typ := rest[:sep]
	cmd, attrs, err := parseMaybeJSON(rest[next:])
	if err != nil {
		return nil, nil, err
	}

	return &Node{Value: typ, Next: cmd, Attributes: attrs}, nil, err
}

func anagram(word1 string, word2 string) bool {
	// make a map containing the number of appearances for each rune
	// (go's encoding-agnostic abstraction of characters)
	// in both strings, and compare them: if they match, then word1 and
	// word2 are anagrams of each other

	// initialize empty maps/dictionaries/hashes that map runes to
	// integers; these are our rune-count dicts for each word
	chars1 := make(map[rune]int)
	chars2 := make(map[rune]int)

	// range gives (int-index, rune) pairs for strings: this is a foreach
	// loop
	for _, c := range strings.ToLower(word1) {
		// discarding spaces makes the function more flexible, so
		// it can check whether two PHRASES, not just two WORDS,
		// are anagrams of each other
		if !unicode.IsSpace(c) {
			// default int value in golang is 0, so this is safe
			chars1[c] = chars1[c] + 1
		}
	}

	for _, c := range strings.ToLower(word2) {
		if !unicode.IsSpace(c) {
			chars2[c] = chars2[c] + 1
		}
	}

	return reflect.DeepEqual(chars1, chars2)
}

func (self *TextPreview) Render(ctx *Context) (err error) {
	if len(self.PlainText) < self.MaxLength {
		ctx.Response.XML.Content(self.PlainText)
	} else {
		shortLength := self.ShortLength
		if shortLength == 0 {
			shortLength = self.MaxLength
		}

		// If in the middle of a word, go back to space before it
		for shortLength > 0 && !unicode.IsSpace(rune(self.PlainText[shortLength-1])) {
			shortLength--
		}

		// If in the middle of space, go back to word before it
		for shortLength > 0 && unicode.IsSpace(rune(self.PlainText[shortLength-1])) {
			shortLength--
		}

		ctx.Response.XML.Content(self.PlainText[:shortLength])
		ctx.Response.XML.Content("... ")
		if self.MoreLink != nil {
			ctx.Response.XML.OpenTag("a")
			ctx.Response.XML.Attrib("href", self.MoreLink.URL(ctx))
			ctx.Response.XML.AttribIfNotDefault("title", self.MoreLink.LinkTitle(ctx))
			content := self.MoreLink.LinkContent(ctx)
			if content != nil {
				err = content.Render(ctx)
			}
			ctx.Response.XML.CloseTagAlways() // a
		}
	}
	return err
}

func count(in *bufio.Reader) (nl, nw, nr, nc int, err error) {
	inword := false
	for {
		var r rune
		var sz int
		r, sz, err = in.ReadRune()
		if err == io.EOF {
			err = nil
			break
		}
		if err != nil {
			return
		}

		nr++
		nc += sz
		if r == '\n' {
			nl++
		}
		if unicode.IsSpace(r) && inword {
			inword = false
			nw++
		} else if !unicode.IsSpace(r) {
			inword = true
		}
	}
	return
}

func lexPrivmsg(l *LogLexer) stateFn {

	for i := 0; ; i++ {
		l.buf.ignoreWhile(func(r rune) bool {
			return unicode.IsSpace(r) && r != '\n'
		})

		n := l.buf.acceptWhile(func(r rune) bool {
			return r != utf8.RuneError && !unicode.IsSpace(r)
		})
		if n > 0 {
			l.emit(markov.TokWord)
		}

		r := l.buf.peek()
		switch {
		case r == '\n':
			l.emit(markov.TokEOL)
			l.buf.next()
			l.buf.ignoreToken()
			l.newline()
			return lexDate
		case r == utf8.RuneError:
			l.errorfEOFValid(nil)
		}
	}
	panic("not reached")
}