Golang bytes.Runes函数代码示例

/*
FixPrematureTransform - Used by clients to fix incoming and outgoing transforms when local changes
have been applied to a document before being routed through the server.

In order for a client UI to be unblocking it must apply local changes as the user types them before
knowing the correct order of the change. Therefore, it is possible to apply a local change before
receiving incoming transforms that are meant to be applied beforehand.

As a solution to those situations this function allows a client to alter and incoming interations
such that if they were to be applied to the local document after our local change they would result
in the same document. The outgoing transform is also modified for sending out to the server.

It is possible that the local change has already been dispatched to the server, in which case it is
the servers responsibility to fix the transform so that other clients end up at the same result.

NOTE: These fixes do not regard or alter the versions of either transform.
*/
func FixPrematureTransform(unapplied, unsent *OTransform) {
	var before, after *OTransform

	// Order the OTs by position in the document.
	if unapplied.Position < unsent.Position {
		before = unapplied
		after = unsent
	} else {
		before = unsent
		after = unapplied
	}

	// Get insertion lengths (codepoints)
	bInsert, aInsert := bytes.Runes([]byte(before.Insert)), bytes.Runes([]byte(after.Insert))
	bLength, aLength := len(bInsert), len(aInsert)

	if before.Delete == 0 {
		after.Position += bLength
	} else if (before.Delete + before.Position) <= after.Position {
		after.Position += (bLength - before.Delete)
	} else {
		posGap := after.Position - before.Position
		excess := intMax(0, before.Delete-posGap)

		if excess > after.Delete {
			before.Delete += (aLength - after.Delete)
			before.Insert = before.Insert + after.Insert
		} else {
			before.Delete = posGap
		}

		after.Delete = intMax(0, after.Delete-excess)
		after.Position = before.Position + bLength
	}
}

func NewBuffer(r io.Reader) (Buffer, error) {
	var b Buffer
	b.Lines = make([]Line, 0)

	// bytes are read into this from the reader
	rb := make([]byte, READ_BUFFER_SIZE)
	// bytes are queued here until we find EOL
	lb := make([]byte, 0, READ_BUFFER_SIZE)
	for {
		n, err := r.Read(rb)
		for i := 0; i < n; i++ {
			lb = append(lb, rb[i])
			if rb[i] == '\n' {
				l := Line(bytes.Runes(lb))
				b.Lines = append(b.Lines, l)
				lb = make([]byte, 0, READ_BUFFER_SIZE)
			}
		}
		if err != nil && err == io.EOF {
			lb = append(lb, '\n')
			l := Line(bytes.Runes(lb))
			b.Lines = append(b.Lines, l)
			lb = make([]byte, 0, READ_BUFFER_SIZE)
			break
		} else if err != nil {
			return b, err
		}
	}
	return b, nil
}

/*
FixOutOfDateTransform - When a transform created for a specific version is later determined to come
after one or more other transforms it can be fixed. This fix translates the transform such that
being applied in the correct order will preserve the original intention.

In order to apply these fixes this function should be called with the target transform and the
actual versioned transform that the target currently 'believes' it is. So, for example, if the
transform was written for version 7 and was actually 10 you would call FixOutOfDateTransform in this
order:

FixOutOfDateTransform(target, version7)
FixOutOfDateTransform(target, version8)
FixOutOfDateTransform(target, version9)

Once the transform is adjusted through this fix it can be harmlessly dispatched to all other clients
which will end up with the same document as the client that submitted this transform.

NOTE: These fixes do not regard or alter the versions of either transform.
*/
func FixOutOfDateTransform(sub, pre *OTransform) {
	// Get insertion lengths (codepoints)
	subInsert, preInsert := bytes.Runes([]byte(sub.Insert)), bytes.Runes([]byte(pre.Insert))
	subLength, preLength := len(subInsert), len(preInsert)

	if pre.Position <= sub.Position {
		if preLength > 0 && pre.Delete == 0 {
			sub.Position += preLength
		} else if pre.Delete > 0 && (pre.Position+pre.Delete) <= sub.Position {
			sub.Position += (preLength - pre.Delete)
		} else if pre.Delete > 0 && (pre.Position+pre.Delete) > sub.Position {
			overhang := intMin(sub.Delete, (pre.Position+pre.Delete)-sub.Position)
			sub.Delete -= overhang
			sub.Position = pre.Position + preLength
		}
	} else if sub.Delete > 0 && (sub.Position+sub.Delete) > pre.Position {
		posGap := pre.Position - sub.Position
		excess := intMax(0, (sub.Delete - posGap))

		if excess > pre.Delete {
			sub.Delete += (preLength - pre.Delete)

			newInsert := make([]rune, subLength+preLength)
			copy(newInsert[:], subInsert)
			copy(newInsert[subLength:], preInsert)

			sub.Insert = string(newInsert)
		} else {
			sub.Delete = posGap
		}
	}
}

func makeEncoding(decode bool) (encoding map[rune]rune) {
	base64Runes := bytes.Runes([]byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="))
	encojiRunes := bytes.Runes([]byte(":grinning_face::grimacing_face::beaming_face_with_smiling_eyes::face_with_tears_of_joy::grinning_face_with_big_eyes::grinning_face_with_smiling_eyes::grinning_face_with_sweat::grinning_squinting_face::smiling_face_with_halo::winking_face::smiling_face_with_smiling_eyes::face_savoring_food::relieved_face::smiling_face_with_heart-eyes::face_blowing_a_kiss::kissing_face::kissing_face_with_smiling_eyes::kissing_face_with_closed_eyes::winking_face_with_tongue::squinting_face_with_tongue::face_with_tongue::smiling_face_with_sunglasses::smirking_face::pile_of_poo::ghost::grinning_cat::grinning_cat_with_smiling_eyes::cat_with_tears_of_joy::smiling_cat_with_heart-eyes::dog_face::cat_face::mouse_face::hamster::rabbit_face::bear::panda::koala::tiger_face::cow_face::pig_face::pig_nose::frog::octopus::monkey_face::see-no-evil_monkey::hear-no-evil_monkey::speak-no-evil_monkey::monkey::chicken::penguin::bird::baby_chick::hatching_chick::front-facing_baby_chick::wolf::boar::horse_face::honeybee::bug::snail::lady_beetle::ant::elephant::dolphin::spouting_whale:"))

	if len(base64Runes) != len(encojiRunes) {
		panic("Charsets must be of same length")
	}

	encoding = make(map[rune]rune)

	var from, to []rune

	if decode {
		from = encojiRunes
		to = base64Runes
	} else {
		from = base64Runes
		to = encojiRunes
	}

	for i := 0; i < len(from); i++ {
		encoding[from[i]] = to[i]
	}
	return encoding
}

func (self *Scanner) scanIdentifier() (*Token, error) {
	var (
		buf  bytes.Buffer
		stop bool
	)

	for !stop {
		char := self.read()
		switch {
		case isLowerCaseLetter(char):
			buf.WriteRune(char)
		case char == eof:
			stop = true
		default:
			self.unread()
			stop = true
		}
	}

	lit := buf.String()
	switch lit {
	case "null":
		return &Token{NULL, lit, bytes.Runes(buf.Bytes())}, nil
	case "true":
		return &Token{TRUE, lit, bytes.Runes(buf.Bytes())}, nil
	case "false":
		return &Token{FALSE, lit, bytes.Runes(buf.Bytes())}, nil
	default:
		return &Token{ILLEGAL, lit, bytes.Runes(buf.Bytes())}, nil
	}
}

func (self *Scanner) scanString() (*Token, error) {
	var (
		buf    bytes.Buffer
		screen bool
		stop   bool
	)

	// skip first quote
	char := self.read()
	if char != '"' {
		// todo err here?
		self.unread()
		return &Token{STRING, buf.String(), bytes.Runes(buf.Bytes())}, nil
	}
	buf.WriteRune(char)

	for !stop {
		char := self.read()

		if unicode.IsControl(char) {
			self.unread()
			stop = true
			continue
		}

		if screen {
			screen = false

			switch char {
			case '"', '\\', '/', 'b', 'f', 'n', 'r', 't', 'u':
				buf.WriteRune(char)
			default:
				return nil, fmt.Errorf("unexpected end of input: %q", string(char))
			}

			continue
		}

		switch {
		case char == '\\':
			screen = true
			buf.WriteRune(char)
		case char == '"':
			buf.WriteRune(char)
			stop = true
		case char == eof:
			stop = true
		default:
			buf.WriteRune(char)
		}
	}

	return &Token{STRING, buf.String(), bytes.Runes(buf.Bytes())}, nil
}

func (fs *FungeSpace) Load(reader io.Reader) error {
	b, err := ioutil.ReadAll(reader)
	if err != nil {
		return err
	}

	address := fs.funge.Origin()

	for _, r := range bytes.Runes(b) {
		switch r {
		case 012, 015:
			if fs.funge > 1 {
				address = address.Add(fs.funge.Delta(YAxis, Forward))
				address.Set(XAxis, 0)
			}
		case 014:
			if fs.funge > 2 {
				address = address.Add(fs.funge.Delta(ZAxis, Forward))
				address.Set(XAxis, 0)
				address.Set(YAxis, 0)
			} else if fs.funge == 2 {
				address = address.Add(fs.funge.Delta(YAxis, Forward))
				address.Set(XAxis, 0)
			}
		default:
			fs.Put(address, r)
			address = address.Add(fs.funge.Delta(XAxis, Forward))
		}
	}

	return nil
}

func main() {
	h := []byte("Hello, I think you are right.你是对的！")

	fmt.Println(`Say you have a string:`, "\n\t", string(h))
	fmt.Println(`Whose byte array in decimal are:`, "\n\t", h)
	fmt.Println(`So in hex: `, "\n\t", fmt.Sprintf("% x", h))
	hex := fmt.Sprintf("%x", h)
	fmt.Println(`So you see this form a lot:`, "\n\t", hex)

	buffer := new(bytes.Buffer)
	encoder := base64.NewEncoder(base64.URLEncoding, buffer)
	encoder.Write(h)
	encoder.Close()

	fmt.Println(`And base64 of the same data:`, "\n\t", buffer.String())

	var bs []byte
	fmt.Sscanf(hex, "%x", &bs)

	fmt.Println(`And you can parse`, "\n\t", hex, "\n", "into bytes", "\n\t", bs)

	fmt.Println(`Whose runes are:`, "\n\t", bytes.Runes(bs))
	fmt.Println(`And each rune matching one charactor not matter its chinese or english:`, "\n\t", string(bs))

}

// Write appends a byte slice into the view's internal buffer. Because
// View implements the io.Writer interface, it can be passed as parameter
// of functions like fmt.Fprintf, fmt.Fprintln, io.Copy, etc. Clear must
// be called to clear the view's buffer.
func (v *View) Write(p []byte) (n int, err error) {
	v.tainted = true

	for _, ch := range bytes.Runes(p) {
		switch ch {
		case '\n':
			v.lines = append(v.lines, nil)
		case '\r':
			nl := len(v.lines)
			if nl > 0 {
				v.lines[nl-1] = nil
			} else {
				v.lines = make([][]cell, 1)
			}
		default:
			cells := v.parseInput(ch)
			if cells == nil {
				continue
			}

			nl := len(v.lines)
			if nl > 0 {
				v.lines[nl-1] = append(v.lines[nl-1], cells...)
			} else {
				v.lines = append(v.lines, cells)
			}
		}
	}
	return len(p), nil
}

func (b *Buffer) Load(q0 uint, fd *os.File, nulls *int) int {
	if q0 > b.nc {
		panic("internal error: buffer.Load")
	}

	p := make([]byte, MaxBlock+utf8.UTFMax+1)

	m := 0
	n := 1
	q1 := q0
	var err error

	for n > 0 {
		n, err = fd.Read(p[m:])
		if err != nil {
			panic(err)
		}
		m += n
		l := m
		if n > 0 {
			l -= utf8.UTFMax
		}
		r := bytes.Runes(p[:l])
		nr := len(r)
		//nb := len([]byte(string(r)))
		copy(p, p[:m])
		b.Insert(q1, uint(nr), r)
		q1 += uint(nr)
	}
	return int(q1 - q0)
}

// Encode writes the transliteration encoding of data to the stream.
func (enc *Encoder) Encode(data []byte) error {
	r := bytes.Runes(data)
	l := len(r)
	var rPrev, rNext rune
	for i, rCurr := range r {
		if i+1 <= l {
			rNext = r[i]
		} else {
			rNext = 0
		}
		var err error
		if s, ok := enc.tbl[rCurr]; ok {
			if sFix, ok := fixRuleRune(rPrev, rCurr, rNext, enc.sys); ok {
				s = sFix
			}
			_, err = enc.WriteString(s)
			if err != nil {
				return err
			}
		} else {
			_, err = enc.WriteRune(rCurr)
			if err != nil {
				return err
			}
		}
		rPrev = rCurr
	}
	return enc.Flush()
}

func truncate(s string, length int) string {
	runes := bytes.Runes([]byte(s))
	if len(runes) > length {
		return string(runes[:length-3]) + "..."
	}
	return string(runes)
}

func (s *CJKWidthFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
	for _, token := range input {
		runeCount := utf8.RuneCount(token.Term)
		runes := bytes.Runes(token.Term)
		for i := 0; i < runeCount; i++ {
			ch := runes[i]
			if ch >= 0xFF01 && ch <= 0xFF5E {
				// fullwidth ASCII variants
				runes[i] -= 0xFEE0
			} else if ch >= 0xFF65 && ch <= 0xFF9F {
				// halfwidth Katakana variants
				if (ch == 0xFF9E || ch == 0xFF9F) && i > 0 && combine(runes, i, ch) {
					runes = analysis.DeleteRune(runes, i)
					i--
					runeCount = len(runes)
				} else {
					runes[i] = kanaNorm[ch-0xFF65]
				}
			}
		}
		token.Term = analysis.BuildTermFromRunes(runes)
	}

	return input
}

func redirToWs(fd int, ws *websocket.Conn) {
	defer func() {
		if r := recover(); r != nil {
			fmt.Fprintf(os.Stderr, "Error occured: %s\n", r)
			runtime.Goexit()
		}
	}()

	var buf [8192]byte
	start, end, buflen := 0, 0, 0
	for {
		switch nr, er := syscall.Read(fd, buf[start:]); {
		case nr < 0:
			fmt.Fprintf(os.Stderr, "error reading from websocket %d with code %d\n", fd, er)
			return
		case nr == 0: // EOF
			return
		case nr > 0:
			buflen = start + nr
			for end = buflen - 1; end >= 0; end-- {
				if utf8.RuneStart(buf[end]) {
					ch, width := utf8.DecodeRune(buf[end:buflen])
					if ch != utf8.RuneError {
						end += width
					}
					break
				}

				if buflen-end >= 6 {
					fmt.Fprintf(os.Stderr, "Invalid UTF-8 sequence in output")
					end = nr
					break
				}
			}

			runes := bytes.Runes(buf[0:end])
			buf_clean := []byte(string(runes))

			nw, ew := ws.Write(buf_clean[:])
			if ew != nil {
				fmt.Fprintf(os.Stderr, "error writing to websocket with code %s\n", ew)
				return
			}

			if nw != len(buf_clean) {
				fmt.Fprintf(os.Stderr, "Written %d instead of expected %d\n", nw, end)
			}

			start = buflen - end

			if start > 0 {
				// copy remaning read bytes from the end to the beginning of a buffer
				// so that we will get normal bytes
				for i := 0; i < start; i++ {
					buf[i] = buf[end+i]
				}
			}
		}
	}
}

func encodeStringAsASCII(str string) string {
	var output bytes.Buffer
	output.WriteByte('"')
	for _, b := range bytes.Runes([]byte(str)) {
		if b < utf8.RuneSelf {
			switch b {
			case '\\', '"':
				output.WriteByte('\\')
				output.WriteByte(byte(b))
			case '\n':
				output.WriteByte('\\')
				output.WriteByte('n')
			case '\r':
				output.WriteByte('\\')
				output.WriteByte('r')
			case '\t':
				output.WriteByte('\\')
				output.WriteByte('t')
			default:
				if b < 0x20 || b == '<' || b == '>' || b == '&' {
					output.WriteString(`\u00`)
					output.WriteByte(hex[b>>4])
					output.WriteByte(hex[b&0xF])
				} else {
					output.WriteByte(byte(b))
				}
			}
		} else {
			output.WriteString(fmt.Sprintf("\\u%04x", b))
		}
	}
	output.WriteByte('"')
	return output.String()
}