Golang Scanner.TokenText方法代码示例

func cmdFind(root *CmdNode, path string, level int) (*CmdNode, error) {

	var s scanner.Scanner
	s.Error = func(s *scanner.Scanner, msg string) {
		log.Printf("command scan error: %s [%s]", msg, path)
	}
	s.Init(strings.NewReader(path))

	parent := root
	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
		//log.Printf("cmdFind: token: [%s]", s.TokenText())
		label := s.TokenText()
		children := matchChildren(parent.Children, label)
		size := len(children)
		if size < 1 {
			return nil, fmt.Errorf("cmdFind: not found: [%s] under [%s]", label, parent.Path)
		}
		if size > 1 {
			return nil, fmt.Errorf("cmdFind: ambiguous: [%s] under [%s]", label, parent.Path)
		}
		parent = children[0]
	}

	//log.Printf("cmdFind: found [%s] as [%s]", path, parent.Path)

	return parent, nil
}

func Example() {
	const src = `
	// This is scanned code.
	if a > 10 {
		someParsable = text
	}`
	var s scanner.Scanner
	s.Init(strings.NewReader(src))
	var tok rune
	for tok != scanner.EOF {
		tok = s.Scan()
		fmt.Println("At position", s.Pos(), ":", s.TokenText())
	}

	// Output:
	// At position 3:4 : if
	// At position 3:6 : a
	// At position 3:8 : >
	// At position 3:11 : 10
	// At position 3:13 : {
	// At position 4:15 : someParsable
	// At position 4:17 : =
	// At position 4:22 : text
	// At position 5:3 : }
	// At position 5:3 :
}

func TestReduceScanner(t *testing.T) {
	in := make(chan interface{})
	go func() {
		reader := strings.NewReader(mapInput)
		s := scanner.Scanner{}
		s.Init(reader)
		tok := s.Scan()
		for tok != scanner.EOF {
			tok = s.Scan()
			in <- s.TokenText()
		}
		close(in)
	}()
	mapOut := Map(in, func(input interface{}) interface{} {
		key := input.(string)
		if len(key) == 0 {
			return ""
		} else {
			return string(key[0])
		}
	})
	redOut := Reduce(mapOut, func(key interface{}, values []interface{}) interface{} {
		return len(values)
	})
	for tok := range redOut {
		if tok.Value.(int) == 0 {
			t.Error("expect reduce should be positive")
		}
	}
}

func (this limitedStringSpan) scanString(s *scanner.Scanner) (string, error) {
	tok := s.Scan()
	if tok != scanner.EOF && tok == -2 {
		return s.TokenText(), nil
	}
	return "", NewInvalidTypeError()
}

func (t Tokenizer) CreateTokens(request *messaging.ParserRequest) (response messaging.ParserResponse) {

	response.Body = make(map[string]string)
	b := bytes.NewBufferString(request.Query)
	var s scanner.Scanner
	s.Init(b)

	index := 0
	for {
		tok := s.Scan()
		if tok != scanner.EOF {
			response.Body[strconv.Itoa(index)] = s.TokenText()
			index++
		} else {
			break
		}

	}

	if index == 0 {
		response.IsSuccess = false
		response.Message = "Error! Nil Query Allocated!"
	} else {
		response.IsSuccess = false
		response.Message = "Success! Query Tokenized Successfully!"
	}

	return
}

/**
 * This function parses a value string parameter and returns Number value
 * embedded within the string. It returns nil if it doesn't find any
 * Number value in the value string.
 * Example: "some4.56more" would return 4.56
 */
func extractFloat32(value string) Number {
	var sc scanner.Scanner
	var tok rune
	var valFloat64 float64
	var valFloat32 Number
	var err error
	var isFound bool

	if len(value) > 0 {
		sc.Init(strings.NewReader(value))
		sc.Mode = scanner.ScanFloats

		for tok != scanner.EOF {
			tok = sc.Scan()
			// fmt.Println("At position", sc.Pos(), ":", sc.TokenText())
			valFloat64, err = strconv.ParseFloat(sc.TokenText(), 64)
			if err == nil {
				isFound = true
				break
			}
		}
	}

	if isFound {
		valFloat32 = Number(valFloat64)
	}

	return valFloat32
}

// Read contents of a Go file up to the package declaration. This can be used
// to find the the build tags.
func readGoContents(fp string) ([]byte, error) {
	f, err := os.Open(fp)
	defer f.Close()
	if err != nil {
		return []byte{}, err
	}

	var s scanner.Scanner
	s.Init(f)
	var tok rune
	var pos scanner.Position
	for tok != scanner.EOF {
		tok = s.Scan()

		// Getting the token text will skip comments by default.
		tt := s.TokenText()
		// build tags will not be after the package declaration.
		if tt == "package" {
			pos = s.Position
			break
		}
	}

	buf := bytes.NewBufferString("")
	f.Seek(0, 0)
	_, err = io.CopyN(buf, f, int64(pos.Offset))
	if err != nil {
		return []byte{}, err
	}

	return buf.Bytes(), nil
}

func TestMapScanner(t *testing.T) {
	in := make(chan interface{})
	go func() {
		reader := strings.NewReader(mapInput)
		s := scanner.Scanner{}
		s.Init(reader)
		tok := s.Scan()
		for tok != scanner.EOF {
			tok = s.Scan()
			in <- s.TokenText()
		}
		close(in)
	}()
	out := Map(in, func(input interface{}) interface{} {
		key := input.(string)
		if len(key) == 0 {
			return ""
		} else {
			return string(key[0])
		}
	})
	for tok := range out {
		for _, val := range tok.Values {
			if !strings.HasPrefix(val.(string), tok.Key.(string)) {
				t.Error(val.(string) + ` should has "` + tok.Key.(string) + `" prefix`)
			}
		}
	}
}

// gofmtFlags looks for a comment of the form
//
//	//gofmt flags
//
// within the first maxLines lines of the given file,
// and returns the flags string, if any. Otherwise it
// returns the empty string.
func gofmtFlags(filename string, maxLines int) string {
	f, err := os.Open(filename)
	if err != nil {
		return "" // ignore errors - they will be found later
	}
	defer f.Close()

	// initialize scanner
	var s scanner.Scanner
	s.Init(f)
	s.Error = func(*scanner.Scanner, string) {}       // ignore errors
	s.Mode = scanner.GoTokens &^ scanner.SkipComments // want comments

	// look for //gofmt comment
	for s.Line <= maxLines {
		switch s.Scan() {
		case scanner.Comment:
			const prefix = "//gofmt "
			if t := s.TokenText(); strings.HasPrefix(t, prefix) {
				return strings.TrimSpace(t[len(prefix):])
			}
		case scanner.EOF:
			return ""
		}

	}

	return ""
}

// ParseKeySet parses a string of comma separated keys
// made of dot separated sections into a KeySet.
func ParseKeySet(s string) (KeySet, error) {
	sc := new(scanner.Scanner).Init(strings.NewReader(s))
	sc.Mode = scanner.ScanIdents | scanner.ScanStrings | scanner.ScanInts
	var k Key
	var ks KeySet
	for {
		// scan section
		switch sc.Scan() {
		case scanner.String:
			u, _ := strconv.Unquote(sc.TokenText())
			if len(u) == 0 {
				return nil, fmt.Errorf("part of key missing in %q", s)
			}
			k = append(k, u)
		case scanner.Ident, scanner.Int:
			k = append(k, sc.TokenText())
		default:
			return nil, fmt.Errorf("bad formatting in %q", s)
		}
		// scan separator
		switch sc.Scan() {
		case '.':
			continue
		case ',':
			ks = append(ks, k)
			k = nil
		case scanner.EOF:
			return append(ks, k), nil
		default:
			return nil, fmt.Errorf("invalid separator in %q", s)
		}
	}
}

func Parser(s string) []string {
	var items []string
	var sc scanner.Scanner
	sc.Init(strings.NewReader(s))
	var tok rune
	for tok != scanner.EOF {
		tok = sc.Scan()
		item := sc.TokenText()
		if tok != scanner.EOF {
			i := len(items)
			if i > 0 {
				if item == "=" && (items[i-1] == "!" || items[i-1] == ">" || items[i-1] == "<" || items[i-1] == "=") {
					items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
					continue
				}
				if item == "|" && (items[i-1]) == "|" {
					items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
					continue
				}
				if item == "&" && (items[i-1]) == "&" {
					items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
					continue
				}
			}
			items = append(items, item)
		}
	}
	return items
}

// Parse updates an inverted index with the file's words
func (index *InvertedIndex) Parse(filename string) {
	if index.alreadyParsed(filename) {
		return
	}
	file, err := os.Open(filename)
	if err != nil {
		cwd, _ := os.Getwd()
		fmt.Printf("The file named %s doesn't exist, the current directory is %s\n",
			filename, cwd)
		return
	}
	defer file.Close()
	var s scanner.Scanner
	s.Init(file)
	var token rune
	for token != scanner.EOF {
		token = s.Scan()
		tokenText := s.TokenText()
		_, found := index.indexMap[tokenText]
		pos := s.Pos()
		result := Result{File: filename,
			Line:  pos.Line,
			Index: pos.Column - len(tokenText)}
		if !found {
			index.indexMap[tokenText] = []Result{result}
		} else {
			index.indexMap[tokenText] = append(index.indexMap[tokenText],
				result)
		}
	}
	index.files = append(index.files, filename)
}

func TestUnwindScanner(t *testing.T) {
	in := make(chan interface{})
	go func() {
		reader := strings.NewReader(unwindInput)
		s := scanner.Scanner{}
		s.Init(reader)
		tok := s.Scan()
		for tok != scanner.EOF {
			tok = s.Scan()
			in <- s.TokenText()
		}
		close(in)
	}()
	out := Unwind(in, func(input interface{}, output chan<- interface{}) {
		output <- strings.ToLower(input.(string))
		output <- strings.ToUpper(input.(string))
	})
	index := 0
	for tok := range out {
		if index%2 == 0 {
			if strings.ToLower(tok.(string)) != tok.(string) {
				t.Error(tok, " should be lower cased")
			}
		} else {
			if strings.ToUpper(tok.(string)) != tok.(string) {
				t.Error(tok, " should be upper cased")
			}
		}
		index++
	}
}

func main() {
	s := new(scanner.Scanner)
	s.Init(os.Stdin)
	for {
		spew.Printf("> ")
		x := s.Scan()
		spew.Println(x, s.TokenText())
	}
}

// generate cuda wrapper for file.
func cuda2go(fname string) {
	// open cuda file
	f, err := os.Open(fname)
	util.PanicErr(err)
	defer f.Close()

	// read tokens
	var token []string
	var s scanner.Scanner
	s.Init(f)
	tok := s.Scan()
	for tok != scanner.EOF {
		if !filter(s.TokenText()) {
			token = append(token, s.TokenText())
		}
		tok = s.Scan()
	}

	// find function name and arguments
	funcname := ""
	argstart, argstop := -1, -1
	for i := 0; i < len(token); i++ {
		if token[i] == "__global__" {
			funcname = token[i+2]
			argstart = i + 4
		}
		if argstart > 0 && token[i] == ")" {
			argstop = i + 1
			break
		}
	}
	argl := token[argstart:argstop]

	// isolate individual arguments
	var args [][]string
	start := 0
	for i, a := range argl {
		if a == "," || a == ")" {
			args = append(args, argl[start:i])
			start = i + 1
		}
	}

	// separate arg names/types and make pointers Go-style
	argn := make([]string, len(args))
	argt := make([]string, len(args))
	for i := range args {
		if args[i][1] == "*" {
			args[i] = []string{args[i][0] + "*", args[i][2]}
		}
		argt[i] = typemap(args[i][0])
		argn[i] = args[i][1]
	}
	wrapgen(fname, funcname, argt, argn)
}