Golang Scanner.Mode方法代碼示例

// ParseKeySet parses a string of comma separated keys
// made of dot separated sections into a KeySet.
func ParseKeySet(s string) (KeySet, error) {
	sc := new(scanner.Scanner).Init(strings.NewReader(s))
	sc.Mode = scanner.ScanIdents | scanner.ScanStrings | scanner.ScanInts
	var k Key
	var ks KeySet
	for {
		// scan section
		switch sc.Scan() {
		case scanner.String:
			u, _ := strconv.Unquote(sc.TokenText())
			if len(u) == 0 {
				return nil, fmt.Errorf("part of key missing in %q", s)
			}
			k = append(k, u)
		case scanner.Ident, scanner.Int:
			k = append(k, sc.TokenText())
		default:
			return nil, fmt.Errorf("bad formatting in %q", s)
		}
		// scan separator
		switch sc.Scan() {
		case '.':
			continue
		case ',':
			ks = append(ks, k)
			k = nil
		case scanner.EOF:
			return append(ks, k), nil
		default:
			return nil, fmt.Errorf("invalid separator in %q", s)
		}
	}
}

// parseKey reads a private key from r. It returns a map[string]string,
// with the key-value pairs, or an error when the file is not correct.
func parseKey(r io.Reader, file string) (map[string]string, error) {
	var s scanner.Scanner
	m := make(map[string]string)
	c := make(chan lex)
	k := ""
	s.Init(r)
	s.Mode = 0
	s.Whitespace = 0
	// Start the lexer
	go klexer(s, c)
	for l := range c {
		// It should alternate
		switch l.value {
		case _KEY:
			k = l.token
		case _VALUE:
			if k == "" {
				return nil, &ParseError{file, "No key seen", l}
			}
			//println("Setting", strings.ToLower(k), "to", l.token, "b")
			m[strings.ToLower(k)] = l.token
			k = ""
		}
	}
	return m, nil
}

func main() {
	var Stdin = os.NewFile(uintptr(syscall.Stdin), "/dev/stdin")
	var s scanner.Scanner
	s.Init(Stdin)
	s.Mode = scanner.ScanIdents | scanner.ScanFloats | scanner.ScanChars | scanner.ScanStrings | scanner.ScanRawStrings | scanner.ScanComments
	tok := s.Scan()
	for tok != scanner.EOF {
		switch tok {
		case scanner.Ident:
			fmt.Println("Ident")
		case scanner.Int:
			fmt.Println("Int")
		case scanner.Float:
			fmt.Println("Float")
		case scanner.Char:
			fmt.Println("Char")
		case scanner.String:
			fmt.Println("String")
		case scanner.RawString:
			fmt.Println("RawString")
		case scanner.Comment:
			fmt.Println("Comment")
		}
		tok = s.Scan()
	}
}

func format(str, dbname string, tm time.Time) string {
	buf := bytes.Buffer{}

	var s scanner.Scanner
	s.Init(strings.NewReader(str))
	s.Mode = 0
	s.Whitespace = 0
	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
		if tok != '%' {
			buf.WriteRune(tok)
			continue
		}

		switch s := s.Scan(); s {
		case '%':
			buf.WriteRune('%')
		case 'n':
			buf.WriteString(dbname)
		case 'Y', 'y':
			buf.WriteString(strconv.Itoa(tm.Year()))
		case 'm':
			buf.WriteString(strconv.Itoa(int(tm.Month())))
		case 'd':
			buf.WriteString(strconv.Itoa(tm.Day()))
		case 'H':
			buf.WriteString(twodig(strconv.Itoa(tm.Hour())))
		case 'M':
			buf.WriteString(twodig(strconv.Itoa(tm.Minute())))
		case 'S':
			buf.WriteString(twodig(strconv.Itoa(tm.Second())))
		}
	}

	return buf.String()
}

func split(expr string) (keys []string, err error) {

	var msgs []string
	var s scanner.Scanner
	s.Init(strings.NewReader(expr))
	s.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanStrings
	s.Error = func(s *scanner.Scanner, msg string) { msgs = append(msgs, fmt.Sprintf("%s %s", s.Pos(), msg)) }

	key := ""
	keys = []string{}
	for err == nil {
		t := s.Peek()
		// fmt.Printf(">>> %s: %s %s\n", s.Pos(), scanner.TokenString(t), s.TokenText())
		switch t {
		case '[':
			key, err = scanBracketedKey(&s)
		case '.':
			s.Scan()
			continue
		case scanner.EOF:
			goto end
		default:
			key, err = scanKey(&s)
		}
		if len(msgs) > 0 {
			err = errors.New(strings.Join(msgs, "\n"))
		}
		if err == nil {
			keys = append(keys, key)
		}
	}
end:
	return
}

/**
 * This function parses a value string parameter and returns Number value
 * embedded within the string. It returns nil if it doesn't find any
 * Number value in the value string.
 * Example: "some4.56more" would return 4.56
 */
func extractFloat32(value string) Number {
	var sc scanner.Scanner
	var tok rune
	var valFloat64 float64
	var valFloat32 Number
	var err error
	var isFound bool

	if len(value) > 0 {
		sc.Init(strings.NewReader(value))
		sc.Mode = scanner.ScanFloats

		for tok != scanner.EOF {
			tok = sc.Scan()
			// fmt.Println("At position", sc.Pos(), ":", sc.TokenText())
			valFloat64, err = strconv.ParseFloat(sc.TokenText(), 64)
			if err == nil {
				isFound = true
				break
			}
		}
	}

	if isFound {
		valFloat32 = Number(valFloat64)
	}

	return valFloat32
}

// gofmtFlags looks for a comment of the form
//
//	//gofmt flags
//
// within the first maxLines lines of the given file,
// and returns the flags string, if any. Otherwise it
// returns the empty string.
func gofmtFlags(filename string, maxLines int) string {
	f, err := os.Open(filename)
	if err != nil {
		return "" // ignore errors - they will be found later
	}
	defer f.Close()

	// initialize scanner
	var s scanner.Scanner
	s.Init(f)
	s.Error = func(*scanner.Scanner, string) {}       // ignore errors
	s.Mode = scanner.GoTokens &^ scanner.SkipComments // want comments

	// look for //gofmt comment
	for s.Line <= maxLines {
		switch s.Scan() {
		case scanner.Comment:
			const prefix = "//gofmt "
			if t := s.TokenText(); strings.HasPrefix(t, prefix) {
				return strings.TrimSpace(t[len(prefix):])
			}
		case scanner.EOF:
			return ""
		}

	}

	return ""
}

func NewScanner(src []byte) *scanner.Scanner {
	var s scanner.Scanner
	s.Init(bytes.NewReader(src))
	s.Error = func(_ *scanner.Scanner, _ string) {}
	s.Whitespace = 0
	s.Mode = s.Mode ^ scanner.SkipComments
	return &s
}

// NewScannerReader takes a reader src and creates a Scanner.
func NewScannerReader(src io.Reader) *scanner.Scanner {
	var s scanner.Scanner
	s.Init(src)
	s.Error = func(_ *scanner.Scanner, _ string) {}
	s.Whitespace = 0
	s.Mode = s.Mode ^ scanner.SkipComments
	return &s
}

// New returns new lexer
func New(r io.Reader) *Lex {
	var s scanner.Scanner
	s.Init(r)
	// only scan characters. implement lexer myself.
	s.Mode &^= scanner.ScanChars | scanner.ScanRawStrings
	return &Lex{
		Scanner: &s,
	}
}

func NewParser(r io.Reader, c ParserClient) *Parser {
	s := new(scanner.Scanner).Init(r)
	s.Mode = (scanner.ScanIdents |
		scanner.ScanInts |
		scanner.ScanFloats |
		scanner.SkipComments |
		scanner.ScanStrings)
	return &Parser{
		s: s,
		c: c,
	}
}

// Parse parses a SPARQL query from the reader
func (sparql *SPARQLUpdate) Parse(src io.Reader) error {
	b, _ := ioutil.ReadAll(src)
	s := new(scanner.Scanner).Init(bytes.NewReader(b))
	s.Mode = scanner.ScanIdents | scanner.ScanStrings

	start := 0
	level := 0
	verb := ""
	tok := s.Scan()
	for tok != scanner.EOF {
		switch tok {
		case -2:
			if level == 0 {
				if len(verb) > 0 {
					verb += " "
				}
				verb += s.TokenText()
			}

		case 123: // {
			if level == 0 {
				start = s.Position.Offset
			}
			level++

		case 125: // }
			level--
			if level == 0 {
				query := SPARQLUpdateQuery{
					body:  string(b[start+1 : s.Position.Offset]),
					graph: NewGraph(sparql.baseURI),
					verb:  verb,
				}
				query.graph.Parse(strings.NewReader(query.body), "text/turtle")
				sparql.queries = append(sparql.queries, query)
			}

		case 59: // ;
			if level == 0 {
				verb = ""
			}
		}

		tok = s.Scan()
	}

	return nil
}

// Text satifies the Extractor interface
//
//ReplaceGo is a specialized routine for correcting Golang source
// files.  Currently only checks comments, not identifiers for
// spelling.
//
// Other items:
//   - check strings, but need to ignore
//      * import "statements" blocks
//      * import ( "blocks" )
//   - skip first comment (line 0) if build comment
//
func (p *GolangText) Text(raw []byte) []byte {
	out := bytes.Buffer{}
	s := scanner.Scanner{}
	s.Init(bytes.NewReader(raw))
	s.Error = (func(s *scanner.Scanner, msg string) {})
	s.Mode = scanner.ScanIdents | scanner.ScanFloats | scanner.ScanChars | scanner.ScanStrings | scanner.ScanRawStrings | scanner.ScanComments
	for {
		switch s.Scan() {
		case scanner.Comment:
			out.WriteString(s.TokenText())
			out.WriteByte('\n')
		case scanner.EOF:
			return out.Bytes()
		}
	}
}

func ScanText(file string) {
	text, err := os.Open(file)
	if err != nil {
		log.Fatal(err)
	}
	defer text.Close()
	var s scanner.Scanner
	s.Init(text)
	s.Mode = scanner.ScanIdents
	tok := s.Scan()
	for tok != scanner.EOF {
		// do something with tok
		doSomething(s.TokenText())
		tok = s.Scan()
	}
	doSomething(nonword)
}

func (theParser *CommandParser) golangTokenizer(line string) []*PreToken {
	var theScanner scanner.Scanner
	result := []*PreToken{}
	theScanner.Init(strings.NewReader(line))
	theScanner.Mode = scanner.ScanFloats | scanner.ScanIdents | scanner.ScanInts | scanner.ScanStrings
	tok := theScanner.Scan()
	for tok != scanner.EOF && tok != COMMENTCHAR {
		s := theScanner.TokenText()
		theToken := &PreToken{
			Type:     tok,
			Text:     s,
			Position: theScanner.Position,
		}
		result = append(result, theToken)
		tok = theScanner.Scan()
	}
	return result
}