Golang scanner.Scanner类代码示例

func (s *PGNSuite) TestPGNParseInfiniteLoopF4(c *C) {
	pgnstr := `[Event "BKL-Turnier"]
[Site "Leipzig"]
[Date "1984.??.??"]
[Round "5"]
[White "Polgar, Zsuzsa"]
[Black "Moehring, Guenther"]
[Result "1-0"]
[WhiteElo "2275"]
[BlackElo "2395"]
[ECO "A49"]

1.d4 Nf6 2.Nf3 d6 3.b3 g6 4.Bb2 Bg7 5.g3 c5 6.Bg2 cxd4 7.Nxd4 d5 8.O-O O-O
9.Na3 Re8 10.Nf3 Nc6 11.c4 dxc4 12.Nxc4 Be6 13.Rc1 Rc8 14.Nfe5 Nxe5 15.Bxe5 Bxc4
16.Rxc4 Rxc4 17.bxc4 Qa5 18.Bxf6 Bxf6 19.Bxb7 Rd8 20.Qb3 Rb8 21.e3 h5 22.Rb1 h4
23.Qb5 Qc7 24.a4 hxg3 25.hxg3 Be5 26.Kg2 Bd6 27.a5 Bc5 28.a6 Rd8 29.Qc6 Qxc6+
30.Bxc6 Rd2 31.Kf3 Rc2 32.Rb8+ Kg7 33.Bb5 Kf6 34.Rc8 Bb6 35.Ba4 Ra2 36.Bb5 Rc2
37.Ke4 e6 38.Kd3 Rc1 39.Kd2 Rb1 40.Kc2 Rb4 41.Rb8 Bc5 42.Rc8 Bb6 43.Rc6 Ba5
44.Rd6 g5 45.f4 gxf4 46.gxf4 Kf5 47.Rd7 Bb6 48.Rxf7+ Ke4 49.Rb7 Bc5 50.Kc3 Kxe3
51.Rc7 Bb6 52.Rc6 Ba5 53.Kc2 Kxf4 54.Rxe6 Bd8 55.Kc3 Rb1 56.Kd4 Rd1+ 57.Kc5 Kf5
58.Re8 Bb6+ 59.Kc6 Kf6 60.Kb7 Bg1 61.Ra8 Re1 62.Rf8+ Kg7 63.Rf5 Kg6 64.Rd5 Rc1
65.Ka8 Be3 66.Rd6+ Kf5 67.Rd3 Ke4 68.Rxe3+ Kxe3 69.Kxa7 Kd4 70.Kb6 Rg1 71.a7 Rg8
72.Kb7 Rg7+ 73.Kb6  1-0`

	r := strings.NewReader(pgnstr)
	sc := scanner.Scanner{}
	sc.Init(r)
	game, err := ParseGame(&sc)
	c.Assert(err, IsNil)
	//	fmt.Println(game)
	c.Assert(game.Tags["Site"], Equals, "Leipzig")
	c.Assert(len(game.Moves), Equals, 145)
}

func Example() {
	const src = `
	// This is scanned code.
	if a > 10 {
		someParsable = text
	}`
	var s scanner.Scanner
	s.Filename = "example"
	s.Init(strings.NewReader(src))
	var tok rune
	for tok != scanner.EOF {
		tok = s.Scan()
		fmt.Println("At position", s.Pos(), ":", s.TokenText())
	}

	// Output:
	// At position example:3:4 : if
	// At position example:3:6 : a
	// At position example:3:8 : >
	// At position example:3:11 : 10
	// At position example:3:13 : {
	// At position example:4:15 : someParsable
	// At position example:4:17 : =
	// At position example:4:22 : text
	// At position example:5:3 : }
	// At position example:5:3 :
}

func getLineToken(line string) []string {
	var tokenScanner scanner.Scanner

	tokenScanner.Init(strings.NewReader(line))

	return tokenize(&tokenScanner)
}

// Parse the initial tag
// e.g. `div`
func ParseTagMatcher(selector *CSSSelector, s scanner.Scanner) error {
	tag := bytes.NewBuffer([]byte{})
	defer func() {
		selector.Tag = tag.String()
	}()
	for {
		c := s.Next()
		switch c {
		case scanner.EOF:
			return nil
		case '.':
			return ParseClassMatcher(selector, s)
		case '#':
			return ParseIdMatcher(selector, s)
		case '[':
			return ParseAttrMatcher(selector, s)
		case ':':
			return ParsePseudo(selector, s)
		default:
			if _, err := tag.WriteRune(c); err != nil {
				return err
			}
		}
	}
}

func copyUntilDelim(in *scanner.Scanner, out *bufio.Writer) error {
	var err error
	for err == nil && !isEOF(in) && in.Peek() != lDelim && in.Peek() != rDelim {
		err = copyNext(in, out)
	}
	return err
}

func (this limitedStringSpan) scanString(s *scanner.Scanner) (string, error) {
	tok := s.Scan()
	if tok != scanner.EOF && tok == -2 {
		return s.TokenText(), nil
	}
	return "", NewInvalidTypeError()
}

// ParseKeySet parses a string of comma separated keys
// made of dot separated sections into a KeySet.
func ParseKeySet(s string) (KeySet, error) {
	sc := new(scanner.Scanner).Init(strings.NewReader(s))
	sc.Mode = scanner.ScanIdents | scanner.ScanStrings | scanner.ScanInts
	var k Key
	var ks KeySet
	for {
		// scan section
		switch sc.Scan() {
		case scanner.String:
			u, _ := strconv.Unquote(sc.TokenText())
			if len(u) == 0 {
				return nil, fmt.Errorf("part of key missing in %q", s)
			}
			k = append(k, u)
		case scanner.Ident, scanner.Int:
			k = append(k, sc.TokenText())
		default:
			return nil, fmt.Errorf("bad formatting in %q", s)
		}
		// scan separator
		switch sc.Scan() {
		case '.':
			continue
		case ',':
			ks = append(ks, k)
			k = nil
		case scanner.EOF:
			return append(ks, k), nil
		default:
			return nil, fmt.Errorf("invalid separator in %q", s)
		}
	}
}

// parseKey reads a private key from r. It returns a map[string]string,
// with the key-value pairs, or an error when the file is not correct.
func parseKey(r io.Reader, file string) (map[string]string, error) {
	var s scanner.Scanner
	m := make(map[string]string)
	c := make(chan lex)
	k := ""
	s.Init(r)
	s.Mode = 0
	s.Whitespace = 0
	// Start the lexer
	go klexer(s, c)
	for l := range c {
		// It should alternate
		switch l.value {
		case _KEY:
			k = l.token
		case _VALUE:
			if k == "" {
				return nil, &ParseError{file, "No key seen", l}
			}
			//println("Setting", strings.ToLower(k), "to", l.token, "b")
			m[strings.ToLower(k)] = l.token
			k = ""
		}
	}
	return m, nil
}

func scanArray(s *scanner.Scanner, out chan Part, stop villa.Stop) (toStop bool) {
	if scanRune(s, out, stop, TP_ARRAY_START, '[') {
		return true
	}

	skipWhitespaces(s)
	if s.Peek() != ']' {
		for {
			if scanValue(s, out, stop) {
				return true
			}

			skipWhitespaces(s)
			if s.Peek() != ',' {
				break
			}

			if scanRune(s, out, stop, TP_COMMA, ',') {
				return true
			}

			skipWhitespaces(s)
		}
	}

	return scanRune(s, out, stop, TP_ARRAY_END, ']')
}

func NewDecoder(r io.Reader) *Decoder {
	var scan scanner.Scanner
	scan.Init(r)
	dec := Decoder{scan: scan}
	scan.Error = dec.setError
	return &dec
}

// Parse an id matcher
// e.g. `#my-picture`
func ParseIdMatcher(selector *CSSSelector, s scanner.Scanner) error {
	var id bytes.Buffer
	defer func() {
		regexpStr := `^` + regexp.QuoteMeta(id.String()) + `$`
		selector.Attrs["id"] = regexp.MustCompile(regexpStr)
	}()
	for {
		c := s.Next()
		switch c {
		case scanner.EOF:
			return nil
		case '.':
			return ParseClassMatcher(selector, s)
		case '#':
			return ParseIdMatcher(selector, s)
		case '[':
			return ParseAttrMatcher(selector, s)
		case ':':
			return ParsePseudo(selector, s)
		default:
			if _, err := id.WriteRune(c); err != nil {
				return err
			}
		}
	}
}

func cmdFind(root *CmdNode, path string, level int) (*CmdNode, error) {

	var s scanner.Scanner
	s.Error = func(s *scanner.Scanner, msg string) {
		log.Printf("command scan error: %s [%s]", msg, path)
	}
	s.Init(strings.NewReader(path))

	parent := root
	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
		//log.Printf("cmdFind: token: [%s]", s.TokenText())
		label := s.TokenText()
		children := matchChildren(parent.Children, label)
		size := len(children)
		if size < 1 {
			return nil, fmt.Errorf("cmdFind: not found: [%s] under [%s]", label, parent.Path)
		}
		if size > 1 {
			return nil, fmt.Errorf("cmdFind: ambiguous: [%s] under [%s]", label, parent.Path)
		}
		parent = children[0]
	}

	//log.Printf("cmdFind: found [%s] as [%s]", path, parent.Path)

	return parent, nil
}

// Parse the selector after ':'
func ParsePseudo(selector *CSSSelector, s scanner.Scanner) error {
	if selector.Pseudo != nil {
		return fmt.Errorf("Combined multiple pseudo classes")
	}
	var b bytes.Buffer
	for s.Peek() != scanner.EOF {
		if _, err := b.WriteRune(s.Next()); err != nil {
			return err
		}
	}
	cmd := b.String()
	var err error
	switch {
	case cmd == "empty":
		selector.Pseudo = func(n *html.Node) bool {
			return n.FirstChild == nil
		}
	case cmd == "first-child":
		selector.Pseudo = firstChildPseudo
	case cmd == "last-child":
		selector.Pseudo = lastChildPseudo
	case cmd == "only-child":
		selector.Pseudo = func(n *html.Node) bool {
			return firstChildPseudo(n) && lastChildPseudo(n)
		}
	case cmd == "first-of-type":
		selector.Pseudo = firstOfTypePseudo
	case cmd == "last-of-type":
		selector.Pseudo = lastOfTypePseudo
	case cmd == "only-of-type":
		selector.Pseudo = func(n *html.Node) bool {
			return firstOfTypePseudo(n) && lastOfTypePseudo(n)
		}
	case strings.HasPrefix(cmd, "contains("):
		selector.Pseudo, err = parseContainsPseudo(cmd[len("contains("):])
		if err != nil {
			return err
		}
	case strings.HasPrefix(cmd, "nth-child("),
		strings.HasPrefix(cmd, "nth-last-child("),
		strings.HasPrefix(cmd, "nth-last-of-type("),
		strings.HasPrefix(cmd, "nth-of-type("):
		if selector.Pseudo, err = parseNthPseudo(cmd); err != nil {
			return err
		}
	case strings.HasPrefix(cmd, "not("):
		if selector.Pseudo, err = parseNotPseudo(cmd[len("not("):]); err != nil {
			return err
		}
	case strings.HasPrefix(cmd, "parent-of("):
		if selector.Pseudo, err = parseParentOfPseudo(cmd[len("parent-of("):]); err != nil {
			return err
		}
	default:
		return fmt.Errorf("%s not a valid pseudo class", cmd)
	}
	return nil
}

func resolveImportsRec(asts []ast, path string, imported []string,
	download bool) ([]ast, error) {
	var newAsts []ast
	top := true // Imports are required to be at the top of the file.

	for _, ast := range asts {
		name := parseImport(ast)
		if name == "" {
			newAsts = append(newAsts, ast)
			top = false
			continue
		}

		if !top {
			return nil, errors.New("import must be begin the module")
		}

		// Check for any import cycles.
		for _, importedModule := range imported {
			if name == importedModule {
				return nil, fmt.Errorf("import cycle: %s",
					append(imported, name))
			}
		}

		modulePath := filepath.Join(path, name+".spec")
		var sc scanner.Scanner
		sc.Filename = modulePath
		if _, err := os.Stat(modulePath); os.IsNotExist(err) && download {
			GetSpec(name)
		}

		f, err := util.Open(modulePath)
		if err != nil {
			return nil, fmt.Errorf("unable to open import %s", name)
		}

		defer f.Close()
		sc.Init(bufio.NewReader(f))
		parsed, err := parse(sc)
		if err != nil {
			return nil, err
		}

		// Rename module name to last name in import path
		name = filepath.Base(name)
		parsed, err = resolveImportsRec(parsed, path, append(imported, name),
			download)
		if err != nil {
			return nil, err
		}

		module := astModule{body: parsed, moduleName: astString(name)}
		newAsts = append(newAsts, module)
	}

	return newAsts, nil
}

// Skip N tokens, if possible. Returns true if it worked out.
func Skip(tokenizer *scanner.Scanner, n int) bool {
	for counter := 0; counter < n; counter++ {
		toktype := tokenizer.Next()
		if toktype == scanner.EOF {
			return false
		}
	}
	return true
}