Golang scanner.Scanner类代码示例

// newScanner creates a new scanner that scans that given input bytes.
func newScanner(input []byte) (*scanner.Scanner, *scanner.ErrorVector) {
	sc := new(scanner.Scanner)
	ev := new(scanner.ErrorVector)
	file := fset.AddFile("input", fset.Base(), len(input))
	sc.Init(file, input, ev, 0)
	return sc, ev
}

func (this *tokenizedGoContent) Update() {
	content := this.GetSources()[0].(caret.MultilineContentI)

	this.segments = nil

	src := []byte(content.Content())
	//src := []byte(w.Content.Content()[w.Content.Line(beginLineIndex).Start:w.Content.Line(endLineIndex).Start])

	var s scanner.Scanner
	fset := token.NewFileSet()
	file := fset.AddFile("", fset.Base(), len(src))
	s.Init(file, src, nil, scanner.ScanComments)

	// Repeated calls to Scan yield the token sequence found in the input.
	// TODO: Perhaps include whitespace in between tokens?
	for {
		pos, tok, lit := s.Scan()
		if tok == token.EOF {
			break
		}

		offset := uint32(fset.Position(pos).Offset)

		this.segments = append(this.segments, tokLit{offset: offset, tok: tok, lit: lit})
	}

	// HACK: Fake last element.
	this.segments = append(this.segments, tokLit{offset: uint32(content.LenContent())})
}

// Scan scans the specified Go source file and returns a channel with Token.
//
// The EOF token is not returned, and the last token does not contain the "\n"
// character.
func Scan(name string, input []byte) chan *Token {
	var s scanner.Scanner

	fset := token.NewFileSet()
	file := fset.AddFile(name, fset.Base(), len(input))
	tokens := make(chan *Token)
	out := make(chan *Token)

	s.Init(file, input, nil, scanner.ScanComments)
	l := lexer{
		input:  string(input),
		file:   file,
		s:      s,
		tokens: tokens,
		out:    out,
	}

	// In the first stage we collect tokens, their literal code and their
	// offset in the source code.
	go l.run1()

	// In the second stage we add white space after each token.
	go l.run2()

	return out
}

// newScanner creates a new scanner that scans that given input bytes.
func newScanner(input []byte) (*scanner.Scanner, *scanner.ErrorVector) {
	sc := new(scanner.Scanner)
	ev := new(scanner.ErrorVector)
	sc.Init("input", input, ev, 0)

	return sc, ev
}

// expectedErrors collects the regular expressions of ERROR comments found
// in files and returns them as a map of error positions to error messages.
//
func expectedErrors(t *testing.T, filename string, src []byte) map[token.Pos]string {
	errors := make(map[token.Pos]string)

	var s scanner.Scanner
	// file was parsed already - do not add it again to the file
	// set otherwise the position information returned here will
	// not match the position information collected by the parser
	s.Init(getFile(filename), src, nil, scanner.ScanComments)
	var prev token.Pos // position of last non-comment, non-semicolon token

	for {
		pos, tok, lit := s.Scan()
		switch tok {
		case token.EOF:
			return errors
		case token.COMMENT:
			s := errRx.FindStringSubmatch(lit)
			if len(s) == 2 {
				errors[prev] = string(s[1])
			}
		default:
			prev = pos
		}
	}

	panic("unreachable")
}

func newTokenIterator(src []byte, cursor int) (tokenIterator, int) {
	fset := token.NewFileSet()
	file := fset.AddFile("", fset.Base(), len(src))
	cursorPos := file.Pos(cursor)

	var s scanner.Scanner
	s.Init(file, src, nil, 0)
	tokens := make([]tokenItem, 0, 1000)
	lastPos := token.NoPos
	for {
		pos, tok, lit := s.Scan()
		if tok == token.EOF || pos >= cursorPos {
			break
		}
		tokens = append(tokens, tokenItem{
			tok: tok,
			lit: lit,
		})
		lastPos = pos
	}
	return tokenIterator{
		tokens: tokens,
		pos:    len(tokens) - 1,
	}, int(cursorPos - lastPos)
}

func formatCode(src []byte, annotations []doc.TypeAnnotation) string {

	// Collect comment positions in type annotation with Name = ""
	var (
		comments []doc.TypeAnnotation
		s        scanner.Scanner
	)
	fset := token.NewFileSet()
	file := fset.AddFile("", fset.Base(), len(src))
	s.Init(file, src, nil, scanner.ScanComments)
commentLoop:
	for {
		pos, tok, lit := s.Scan()
		switch tok {
		case token.EOF:
			break commentLoop
		case token.COMMENT:
			p := file.Offset(pos)
			comments = append(comments, doc.TypeAnnotation{Pos: p, End: p + len(lit)})
		}
	}

	// Merge type annotations and comments without modifying the caller's slice
	// of annoations.
	switch {
	case len(comments) == 0:
		// nothing to do
	case len(annotations) == 0:
		annotations = comments
	default:
		annotations = append(comments, annotations...)
		sort.Sort(sortByPos(annotations))
	}

	var buf bytes.Buffer
	last := 0
	for _, a := range annotations {
		template.HTMLEscape(&buf, src[last:a.Pos])
		if a.Name != "" {
			p := a.ImportPath
			if p != "" {
				p = "/" + p
			}
			buf.WriteString(`<a href="`)
			buf.WriteString(urlFn(p))
			buf.WriteByte('#')
			buf.WriteString(urlFn(a.Name))
			buf.WriteString(`">`)
			template.HTMLEscape(&buf, src[a.Pos:a.End])
			buf.WriteString(`</a>`)
		} else {
			buf.WriteString(`<span class="com">`)
			template.HTMLEscape(&buf, src[a.Pos:a.End])
			buf.WriteString(`</span>`)
		}
		last = a.End
	}
	template.HTMLEscape(&buf, src[last:])
	return buf.String()
}

func main2() {

	filename := "/Users/obaskakov/IdeaProjects/goCrazy/code.scm"
	src, err := ioutil.ReadFile(filename)
	check(err)

	//	src := []byte("cos(x) + 1i*sin(x) // Euler")

	// Initialize the scanner.
	var s scanner.Scanner
	fset := token.NewFileSet()
	file := fset.AddFile("", fset.Base(), len(src)) // register input "file"
	s.Init(file, src, nil /* no error handler */, scanner.ScanComments)

	// Repeated calls to Scan yield the token sequence found in the input.
	for {
		_, tok, lit := s.Scan()
		if tok == token.EOF {
			break
		}
		fmt.Printf("\t%s    %q\n", tok, lit)
		//		fmt.Printf("%s\t%s\t%q\n", fset.Position(pos), tok, lit)
	}

}

// firstComment returns the contents of the first comment in
// the given file, assuming there's one within the first KB.
func firstComment(filename string) string {
	f, err := os.Open(filename)
	if err != nil {
		return ""
	}
	defer f.Close()

	var src [1 << 10]byte // read at most 1KB
	n, _ := f.Read(src[:])

	var s scanner.Scanner
	s.Init(fset.AddFile("", fset.Base(), n), src[:n], nil, scanner.ScanComments)
	for {
		_, tok, lit := s.Scan()
		switch tok {
		case token.COMMENT:
			// remove trailing */ of multi-line comment
			if lit[1] == '*' {
				lit = lit[:len(lit)-2]
			}
			return strings.TrimSpace(lit[2:])
		case token.EOF:
			return ""
		}
	}
}

// expectedErrors collects the regular expressions of ERROR comments
// found in the package files of pkg and returns them in sorted order
// (by filename and position).
func expectedErrors(t *testing.T, pkg *ast.Package) (list scanner.ErrorList) {
	// scan all package files
	for filename := range pkg.Files {
		src, err := ioutil.ReadFile(filename)
		if err != nil {
			t.Fatalf("expectedErrors(%s): %v", pkg.Name, err)
		}

		var s scanner.Scanner
		file := fset.AddFile(filename, fset.Base(), len(src))
		s.Init(file, src, nil, scanner.ScanComments)
		var prev token.Pos // position of last non-comment token
	loop:
		for {
			pos, tok, lit := s.Scan()
			switch tok {
			case token.EOF:
				break loop
			case token.COMMENT:
				s := errRx.FindStringSubmatch(lit)
				if len(s) == 2 {
					list = append(list, &scanner.Error{fset.Position(prev), string(s[1])})
				}
			default:
				prev = pos
			}
		}
	}
	sort.Sort(list) // multiple files may not be sorted
	return
}

func new_token_iterator(src []byte, cursor int) token_iterator {
	tokens := make([]token_item, 0, 1000)
	var s scanner.Scanner
	fset := token.NewFileSet()
	file := fset.AddFile("", fset.Base(), len(src))
	s.Init(file, src, nil, 0)
	token_index := 0
	for {
		pos, tok, lit := s.Scan()
		if tok == token.EOF {
			break
		}
		off := fset.Position(pos).Offset
		tokens = append(tokens, token_item{
			off: off,
			tok: tok,
			lit: lit,
		})
		if cursor > off {
			token_index++
		}
	}
	return token_iterator{
		tokens:      tokens,
		token_index: token_index,
	}
}

func NewToken(src string, cursor int) tokenIterator {
	src = src[:cursor] + ";" + src[cursor:]
	tokens := make([]tokenItem, 0, 1000)
	var s scanner.Scanner
	fset := token.NewFileSet()
	file := fset.AddFile("", fset.Base(), len(src))
	s.Init(file, []byte(src), nil, 0)
	index := 0
	for {
		pos, tok, lit := s.Scan()
		if tok == token.EOF {
			break
		}
		off := fset.Position(pos).Offset

		tokens = append(tokens, tokenItem{
			off: off,
			tok: tok,
			lit: lit,
		})
		if cursor > off {
			index++
		}
	}
	return tokenIterator{
		tokens: tokens,
		index:  index,
	}
}

func ExampleScanner_Scan() {
	// src is the input that we want to tokenize.
	src := []byte("cos(x) + 1i*sin(x) // Euler")

	// Initialize the scanner.
	var s scanner.Scanner
	fset := token.NewFileSet()                      // positions are relative to fset
	file := fset.AddFile("", fset.Base(), len(src)) // register input "file"
	s.Init(file, src, nil /* no error handler */, scanner.ScanComments)

	// Repeated calls to Scan yield the token sequence found in the input.
	for {
		pos, tok, lit := s.Scan()
		if tok == token.EOF {
			break
		}
		fmt.Printf("%s\t%s\t%q\n", fset.Position(pos), tok, lit)
	}

	// output:
	// 1:1	IDENT	"cos"
	// 1:4	(	""
	// 1:5	IDENT	"x"
	// 1:6	)	""
	// 1:8	+	""
	// 1:10	IMAG	"1i"
	// 1:12	*	""
	// 1:13	IDENT	"sin"
	// 1:16	(	""
	// 1:17	IDENT	"x"
	// 1:18	)	""
	// 1:20	;	"\n"
	// 1:20	COMMENT	"// Euler"
}

func getTypes(s *scanner.Scanner) (params []string, types []string, pos token.Pos, error error) {
	tok := token.COMMA
	var lit string
	for tok == token.COMMA {
		pos, tok, lit = s.Scan()
		if tok != token.TYPE {
			error = errors.New("Expected 'type', not " + lit)
			return
		}
		var tname string
		var par string
		pos, tok, par = s.Scan()
		if tok != token.IDENT {
			error = errors.New("Identifier expected, not " + string(par))
			return
		}
		tname, pos, tok, lit = getType(s)
		params = append(params, string(par))
		types = append(types, string(tname))
	}
	if tok != token.RPAREN {
		error = errors.New(fmt.Sprintf("inappropriate token %v with lit: %s",
			tok, lit))
	}
	return
}

// expectedErrors collects the regular expressions of ERROR comments found
// in files and returns them as a map of error positions to error messages.
//
func expectedErrors(t *testing.T, testname string, files map[string]*ast.File) map[token.Pos]string {
	errors := make(map[token.Pos]string)
	for filename := range files {
		src, err := ioutil.ReadFile(filename)
		if err != nil {
			t.Fatalf("%s: could not read %s", testname, filename)
		}

		var s scanner.Scanner
		// file was parsed already - do not add it again to the file
		// set otherwise the position information returned here will
		// not match the position information collected by the parser
		s.Init(getFile(filename), src, nil, scanner.ScanComments)
		var prev token.Pos // position of last non-comment token

	scanFile:
		for {
			pos, tok, lit := s.Scan()
			switch tok {
			case token.EOF:
				break scanFile
			case token.COMMENT:
				s := errRx.FindStringSubmatch(lit)
				if len(s) == 2 {
					errors[prev] = string(s[1])
				}
			default:
				prev = pos
			}
		}
	}
	return errors
}