Golang Scanner.Scan方法代码示例

func main() {
	var Stdin = os.NewFile(uintptr(syscall.Stdin), "/dev/stdin")
	var s scanner.Scanner
	s.Init(Stdin)
	s.Mode = scanner.ScanIdents | scanner.ScanFloats | scanner.ScanChars | scanner.ScanStrings | scanner.ScanRawStrings | scanner.ScanComments
	tok := s.Scan()
	for tok != scanner.EOF {
		switch tok {
		case scanner.Ident:
			fmt.Println("Ident")
		case scanner.Int:
			fmt.Println("Int")
		case scanner.Float:
			fmt.Println("Float")
		case scanner.Char:
			fmt.Println("Char")
		case scanner.String:
			fmt.Println("String")
		case scanner.RawString:
			fmt.Println("RawString")
		case scanner.Comment:
			fmt.Println("Comment")
		}
		tok = s.Scan()
	}
}

func Tokenize(text string, stopwords map[string]int, words map[string]int) {
	var s scanner.Scanner
	s.Init(strings.NewReader(text))
	tok := s.Scan()
	for tok != scanner.EOF {
		if tok == scanner.String {
			Tokenize(strings.Trim(s.TokenText(), "\"`"), stopwords, words)
		} else if tok == scanner.Char {
			Tokenize(strings.Trim(s.TokenText(), "'"), stopwords, words)
		} else if tok == scanner.Ident {
			word := s.TokenText()
			if _, ok := stopwords[word]; !ok && len(word) > 2 {
				stem, err := snowball.Stem(word, "english", true)
				if err != nil {
					fmt.Errorf("Couldnt stem word: %s", word)
					stem = word
				}
				if _, ok := stopwords[stem]; !ok {
					words[stem] += 1
				}
			}
		}
		tok = s.Scan()
	}
}

// gofmtFlags looks for a comment of the form
//
//	//gofmt flags
//
// within the first maxLines lines of the given file,
// and returns the flags string, if any. Otherwise it
// returns the empty string.
func gofmtFlags(filename string, maxLines int) string {
	f, err := os.Open(filename)
	if err != nil {
		return "" // ignore errors - they will be found later
	}
	defer f.Close()

	// initialize scanner
	var s scanner.Scanner
	s.Init(f)
	s.Error = func(*scanner.Scanner, string) {}       // ignore errors
	s.Mode = scanner.GoTokens &^ scanner.SkipComments // want comments

	// look for //gofmt comment
	for s.Line <= maxLines {
		switch s.Scan() {
		case scanner.Comment:
			const prefix = "//gofmt "
			if t := s.TokenText(); strings.HasPrefix(t, prefix) {
				return strings.TrimSpace(t[len(prefix):])
			}
		case scanner.EOF:
			return ""
		}

	}

	return ""
}

func Example() {
	const src = `
	// This is scanned code.
	if a > 10 {
		someParsable = text
	}`
	var s scanner.Scanner
	s.Init(strings.NewReader(src))
	var tok rune
	for tok != scanner.EOF {
		tok = s.Scan()
		fmt.Println("At position", s.Pos(), ":", s.TokenText())
	}

	// Output:
	// At position 3:4 : if
	// At position 3:6 : a
	// At position 3:8 : >
	// At position 3:11 : 10
	// At position 3:13 : {
	// At position 4:15 : someParsable
	// At position 4:17 : =
	// At position 4:22 : text
	// At position 5:3 : }
	// At position 5:3 :
}

func cmdFind(root *CmdNode, path string, level int) (*CmdNode, error) {

	var s scanner.Scanner
	s.Error = func(s *scanner.Scanner, msg string) {
		log.Printf("command scan error: %s [%s]", msg, path)
	}
	s.Init(strings.NewReader(path))

	parent := root
	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
		//log.Printf("cmdFind: token: [%s]", s.TokenText())
		label := s.TokenText()
		children := matchChildren(parent.Children, label)
		size := len(children)
		if size < 1 {
			return nil, fmt.Errorf("cmdFind: not found: [%s] under [%s]", label, parent.Path)
		}
		if size > 1 {
			return nil, fmt.Errorf("cmdFind: ambiguous: [%s] under [%s]", label, parent.Path)
		}
		parent = children[0]
	}

	//log.Printf("cmdFind: found [%s] as [%s]", path, parent.Path)

	return parent, nil
}

func TestReduceScanner(t *testing.T) {
	in := make(chan interface{})
	go func() {
		reader := strings.NewReader(mapInput)
		s := scanner.Scanner{}
		s.Init(reader)
		tok := s.Scan()
		for tok != scanner.EOF {
			tok = s.Scan()
			in <- s.TokenText()
		}
		close(in)
	}()
	mapOut := Map(in, func(input interface{}) interface{} {
		key := input.(string)
		if len(key) == 0 {
			return ""
		} else {
			return string(key[0])
		}
	})
	redOut := Reduce(mapOut, func(key interface{}, values []interface{}) interface{} {
		return len(values)
	})
	for tok := range redOut {
		if tok.Value.(int) == 0 {
			t.Error("expect reduce should be positive")
		}
	}
}

func TestMapScanner(t *testing.T) {
	in := make(chan interface{})
	go func() {
		reader := strings.NewReader(mapInput)
		s := scanner.Scanner{}
		s.Init(reader)
		tok := s.Scan()
		for tok != scanner.EOF {
			tok = s.Scan()
			in <- s.TokenText()
		}
		close(in)
	}()
	out := Map(in, func(input interface{}) interface{} {
		key := input.(string)
		if len(key) == 0 {
			return ""
		} else {
			return string(key[0])
		}
	})
	for tok := range out {
		for _, val := range tok.Values {
			if !strings.HasPrefix(val.(string), tok.Key.(string)) {
				t.Error(val.(string) + ` should has "` + tok.Key.(string) + `" prefix`)
			}
		}
	}
}

func split(expr string) (keys []string, err error) {

	var msgs []string
	var s scanner.Scanner
	s.Init(strings.NewReader(expr))
	s.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanStrings
	s.Error = func(s *scanner.Scanner, msg string) { msgs = append(msgs, fmt.Sprintf("%s %s", s.Pos(), msg)) }

	key := ""
	keys = []string{}
	for err == nil {
		t := s.Peek()
		// fmt.Printf(">>> %s: %s %s\n", s.Pos(), scanner.TokenString(t), s.TokenText())
		switch t {
		case '[':
			key, err = scanBracketedKey(&s)
		case '.':
			s.Scan()
			continue
		case scanner.EOF:
			goto end
		default:
			key, err = scanKey(&s)
		}
		if len(msgs) > 0 {
			err = errors.New(strings.Join(msgs, "\n"))
		}
		if err == nil {
			keys = append(keys, key)
		}
	}
end:
	return
}

// Parse updates an inverted index with the file's words
func (index *InvertedIndex) Parse(filename string) {
	if index.alreadyParsed(filename) {
		return
	}
	file, err := os.Open(filename)
	if err != nil {
		cwd, _ := os.Getwd()
		fmt.Printf("The file named %s doesn't exist, the current directory is %s\n",
			filename, cwd)
		return
	}
	defer file.Close()
	var s scanner.Scanner
	s.Init(file)
	var token rune
	for token != scanner.EOF {
		token = s.Scan()
		tokenText := s.TokenText()
		_, found := index.indexMap[tokenText]
		pos := s.Pos()
		result := Result{File: filename,
			Line:  pos.Line,
			Index: pos.Column - len(tokenText)}
		if !found {
			index.indexMap[tokenText] = []Result{result}
		} else {
			index.indexMap[tokenText] = append(index.indexMap[tokenText],
				result)
		}
	}
	index.files = append(index.files, filename)
}

// Read contents of a Go file up to the package declaration. This can be used
// to find the the build tags.
func readGoContents(fp string) ([]byte, error) {
	f, err := os.Open(fp)
	defer f.Close()
	if err != nil {
		return []byte{}, err
	}

	var s scanner.Scanner
	s.Init(f)
	var tok rune
	var pos scanner.Position
	for tok != scanner.EOF {
		tok = s.Scan()

		// Getting the token text will skip comments by default.
		tt := s.TokenText()
		// build tags will not be after the package declaration.
		if tt == "package" {
			pos = s.Position
			break
		}
	}

	buf := bytes.NewBufferString("")
	f.Seek(0, 0)
	_, err = io.CopyN(buf, f, int64(pos.Offset))
	if err != nil {
		return []byte{}, err
	}

	return buf.Bytes(), nil
}

func Parser(s string) []string {
	var items []string
	var sc scanner.Scanner
	sc.Init(strings.NewReader(s))
	var tok rune
	for tok != scanner.EOF {
		tok = sc.Scan()
		item := sc.TokenText()
		if tok != scanner.EOF {
			i := len(items)
			if i > 0 {
				if item == "=" && (items[i-1] == "!" || items[i-1] == ">" || items[i-1] == "<" || items[i-1] == "=") {
					items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
					continue
				}
				if item == "|" && (items[i-1]) == "|" {
					items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
					continue
				}
				if item == "&" && (items[i-1]) == "&" {
					items[i-1] = fmt.Sprintf("%s%s", items[i-1], item)
					continue
				}
			}
			items = append(items, item)
		}
	}
	return items
}

func (t Tokenizer) CreateTokens(request *messaging.ParserRequest) (response messaging.ParserResponse) {

	response.Body = make(map[string]string)
	b := bytes.NewBufferString(request.Query)
	var s scanner.Scanner
	s.Init(b)

	index := 0
	for {
		tok := s.Scan()
		if tok != scanner.EOF {
			response.Body[strconv.Itoa(index)] = s.TokenText()
			index++
		} else {
			break
		}

	}

	if index == 0 {
		response.IsSuccess = false
		response.Message = "Error! Nil Query Allocated!"
	} else {
		response.IsSuccess = false
		response.Message = "Success! Query Tokenized Successfully!"
	}

	return
}

func (this limitedStringSpan) scanString(s *scanner.Scanner) (string, error) {
	tok := s.Scan()
	if tok != scanner.EOF && tok == -2 {
		return s.TokenText(), nil
	}
	return "", NewInvalidTypeError()
}

func TestUnwindScanner(t *testing.T) {
	in := make(chan interface{})
	go func() {
		reader := strings.NewReader(unwindInput)
		s := scanner.Scanner{}
		s.Init(reader)
		tok := s.Scan()
		for tok != scanner.EOF {
			tok = s.Scan()
			in <- s.TokenText()
		}
		close(in)
	}()
	out := Unwind(in, func(input interface{}, output chan<- interface{}) {
		output <- strings.ToLower(input.(string))
		output <- strings.ToUpper(input.(string))
	})
	index := 0
	for tok := range out {
		if index%2 == 0 {
			if strings.ToLower(tok.(string)) != tok.(string) {
				t.Error(tok, " should be lower cased")
			}
		} else {
			if strings.ToUpper(tok.(string)) != tok.(string) {
				t.Error(tok, " should be upper cased")
			}
		}
		index++
	}
}

// ParseKeySet parses a string of comma separated keys
// made of dot separated sections into a KeySet.
func ParseKeySet(s string) (KeySet, error) {
	sc := new(scanner.Scanner).Init(strings.NewReader(s))
	sc.Mode = scanner.ScanIdents | scanner.ScanStrings | scanner.ScanInts
	var k Key
	var ks KeySet
	for {
		// scan section
		switch sc.Scan() {
		case scanner.String:
			u, _ := strconv.Unquote(sc.TokenText())
			if len(u) == 0 {
				return nil, fmt.Errorf("part of key missing in %q", s)
			}
			k = append(k, u)
		case scanner.Ident, scanner.Int:
			k = append(k, sc.TokenText())
		default:
			return nil, fmt.Errorf("bad formatting in %q", s)
		}
		// scan separator
		switch sc.Scan() {
		case '.':
			continue
		case ',':
			ks = append(ks, k)
			k = nil
		case scanner.EOF:
			return append(ks, k), nil
		default:
			return nil, fmt.Errorf("invalid separator in %q", s)
		}
	}
}