C# LexerMode类代码示例

 public static AST.Node LoadFile(string fileName, LexerMode mode, FunctionInformation functionInfo)
 {
     using (StreamReader file = new StreamReader(fileName, Parse.Latin1))
     {
         return Parse.String(file.ReadToEnd(), mode, functionInfo);
     }
 }

 internal ResetPoint(int resetCount, LexerMode mode, int position, CSharpSyntaxNode prevTokenTrailingTrivia)
 {
     this.ResetCount = resetCount;
     this.Mode = mode;
     this.Position = position;
     this.PrevTokenTrailingTrivia = prevTokenTrailingTrivia;
 }

 /// <summary>
 /// 重置分析器
 /// </summary>
 public void Reset()
 {
     this.mode = LexerMode.None;
     this.line = 1;
     this.column = 1;
     this.kind = TokenKind.Text;
     this.startColumn = 1;
     this.startLine = 1;
     this.scanner = new CharScanner(this.document);
     this.collection = new List<Token>();
     this.pos = new Stack<String>();
 }

        public void Next(LexerMode mode)
        {
            switch (mode)
            {
                case LexerMode.BLOCK:
                    this.NextBlock ();

                    break;

                case LexerMode.RAW:
                    this.NextRaw ();

                    break;

                default:
                    throw new UnknownException (this, "invalid lexem");
            }
        }

        public void Next(LexerMode mode)
        {
            switch (mode)
            {
                case LexerMode.Block:
                    this.current = this.NextBlock ();

                    break;

                case LexerMode.Raw:
                    this.current = this.NextRaw ();

                    break;

                default:
                    throw new ParseException (this.column, this.line, "<?>", "block or raw text");
            }
        }

        public static AST.Node String(string input, LexerMode mode, FunctionInformation functionInfo)
        {
            switch (mode)
            {
                case LexerMode.ASCII:
                    return ASCIIString(input, functionInfo);

                case LexerMode.APL:
                    return APLString(input, functionInfo);

                case LexerMode.UNI:
                    return UNIString(input, functionInfo);

                default:
                    break;
            }

            throw new ParseException("Invalid Parse Mode");
        }

        /// <summary>
        /// 分析所有Token
        /// </summary>
        /// <returns></returns>
        public Token[] Parse()
        {
            if (this.kind != TokenKind.EOF)
            {
                do
                {
                    if (this.mode == LexerMode.EnterLabel)
                    {
                        Next(this.pos.Peek().Length - 1);
                        AddToken(GetToken(GetTokenKind(this.scanner.Read())));
                        switch (this.kind)
                        {
                            case TokenKind.StringStart:
                                this.pos.Push("\"");
                                break;
                            case TokenKind.LeftParentheses:
                                this.pos.Push("(");
                                break;
                        }
                        ReadToken();
                    }
                    else if (IsTagStart())
                    {
                        AddToken(GetToken(TokenKind.TagStart));
                        this.mode = LexerMode.EnterLabel;

                    }
                    else if (this.scanner.Read() == '\n')
                    {
                        this.line++;
                        this.column = 1;
                    }
                }
                while (Next());

                AddToken(GetToken(TokenKind.EOF));


                if (this.mode == LexerMode.EnterLabel)
                {
                    this.mode = LexerMode.LeaveLabel;
                    AddToken(new Token(TokenKind.TagEnd, String.Empty));
                }

            }

            return this.collection.ToArray();

        }

        private SyntaxNode LexSingleDirective(
            bool isActive,
            bool endIsActive,
            bool afterNonWhitespaceOnLine,
            List<SyntaxNode> triviaList)
        {
            _start = _charReader.Position;

            if (char.IsWhiteSpace(_charReader.Current))
            {
                ReadWhitespace();
                AddTrivia(triviaList, SyntaxKind.WhitespaceTrivia);
            }

            var saveMode = _mode;
            var saveExpandMacros = ExpandMacros;

            _mode = LexerMode.Directive;
            ExpandMacros = false;

            var dp = new DirectiveParser(this, _directives);
            var directive = dp.ParseDirective(isActive, endIsActive, afterNonWhitespaceOnLine);

            if (!isActive || directive.Kind != SyntaxKind.IncludeDirectiveTrivia)
                triviaList.Add(directive);

            _directives = directive.ApplyDirectives(_directives);
            ExpandMacros = saveExpandMacros;
            _mode = saveMode;

            // Directive parser sometimes leaves charReader at start of token *after* the one we want.
            _charReader.Reset(directive.GetLastToken().GetLastSpanIncludingTrivia().End);
            _start = _charReader.Position;

            return directive;
        }

 private BlendedNode ReadNodeOrToken(LexerMode mode, bool asToken)
 {
     var reader = new Reader(this);
     return reader.ReadNodeOrToken(mode, asToken);
 }

 public BlendedNode ReadToken(LexerMode mode)
 {
     return ReadNodeOrToken(mode, asToken: true);
 }

 public BlendedNode ReadNode(LexerMode mode)
 {
     return ReadNodeOrToken(mode, asToken: false);
 }

            internal BlendedNode ReadNodeOrToken(LexerMode mode, bool asToken)
            {
                // This is the core driver of the blender.  It just sits in a loop trying to keep our
                // positions in the old and new text in sync.  When they're out of sync it will try
                // to match them back up, and it will appropriately determine which nodes or tokens
                // from the old tree can be reused as long as they don't overlap and changes or
                // contain any errors.

                while (true)
                {
                    // If the cursor in the old tree is finished, then our choice is easy.  We just
                    // read from the new text.
                    if (this.oldTreeCursor.IsFinished)
                    {
                        return this.ReadNewToken(mode);
                    }

                    // If delta is non-zero then that means our positions in the respective text
                    // streams are not in sync.  This can be because of to reasons.  Either:
                    //
                    //   a) we're further ahead in the new text (i.e. 'changeDelta' is negative). We
                    //      should keep skipping tokens in the old text until we catch up.
                    //      TODO(cyrusn): We could actually be smarter here and skip whole nodes if
                    //      they're shorter than the changeDelta.  We can try doing that in the future.
                    //
                    //   b) we're further ahead in the old text (i.e. 'changeDelta' is positive).
                    //      This can happen when we are skipping over portions of the old tree because
                    //      it overlapped with changed text spans. In this case, we want to read a
                    //      token to try to consume that changed text and ensure that we get synced up.
                    if (this.changeDelta < 0)
                    {
                        // Case '1' above.  We're behind in the old text, so move forward a token.
                        // And try again.
                        this.SkipOldToken();
                    }
                    else if (this.changeDelta > 0)
                    {
                        // Case '2' above.  We're behind in the new text, so read a token to try to
                        // catch up.
                        return this.ReadNewToken(mode);
                    }
                    else
                    {
                        // Attempt to take a node or token from the old tree.  If we can't, then
                        // either break down the current node we're looking at to its first child
                        // and try again, or move to the next token.
                        BlendedNode blendedNode;
                        if (this.TryTakeOldNodeOrToken(asToken, out blendedNode))
                        {
                            return blendedNode;
                        }

                        // Couldn't take the current node or token.  Figure out the next node or
                        // token to reconsider and try again.
                        if (this.oldTreeCursor.CurrentNodeOrToken.IsNode)
                        {
                            // It was a node.  Just move to its first token and try again.
                            this.oldTreeCursor = this.oldTreeCursor.MoveToFirstChild();
                        }
                        else
                        {
                            // It was a token, just move to the next token.
                            this.SkipOldToken();
                        }
                    }
                }
            }

            private SyntaxToken LexNewToken(LexerMode mode)
            {
                if (this.lexer.TextWindow.Position != this.newPosition)
                {
                    this.lexer.Reset(this.newPosition, this.newDirectives);
                }

                if (mode >= LexerMode.XmlDocComment)
                {
                    mode |= this.newLexerDrivenMode;
                }

                var token = this.lexer.Lex(ref mode);
                this.newDirectives = this.lexer.Directives;
                this.newLexerDrivenMode = mode & (LexerMode.MaskXmlDocCommentLocation | LexerMode.MaskXmlDocCommentStyle);
                return token;
            }

            private BlendedNode ReadNewToken(LexerMode mode)
            {
                Debug.Assert(this.changeDelta > 0 || this.oldTreeCursor.IsFinished);

                // The new text is either behind the cursor, or the cursor is done.  In either event,
                // we need to lex a real token from the stream.
                var token = this.LexNewToken(mode);

                // If the oldTreeCursor was finished, then the below code isn't really necessary.
                // We'll just repeat the outer reader loop and call right back into ReadNewToken.
                // That will then call LexNewToken (which doesn't use either of these variables).  If
                // oldTreeCursor wasn't finished then we need to update our state based on the token
                // we just read.
                var width = token.FullWidth;
                this.newPosition += width;
                this.changeDelta -= width;

                // By reading a token we may either have read into, or past, change ranges.  Skip
                // past them.  This will increase changeDelta which will indicate to us that we need
                // to keep on lexing.
                this.SkipPastChanges();

                return this.CreateBlendedNode(node: null, token: token);
            }

        public SyntaxToken Lex(LexerMode mode)
        {
#if DEBUG
            TokensLexed++;
#endif
            this.mode = mode;
            switch (this.mode)
            {
                case LexerMode.Syntax:
                case LexerMode.DebuggerSyntax:
#if true
                    var result = this.QuickScanSyntaxToken();
                    if (result == null)
                    {
                        result = this.LexSyntaxToken();
                    }

                    return result;
#else
                    return this.LexSyntaxToken();
#endif
                case LexerMode.Directive:
                    return this.LexDirectiveToken();
            }

            switch (ModeOf(this.mode))
            {
                case LexerMode.XmlDocComment:
                    return this.LexXmlToken();
                case LexerMode.XmlElementTag:
                    return this.LexXmlElementTagToken();
                case LexerMode.XmlAttributeTextQuote:
                case LexerMode.XmlAttributeTextDoubleQuote:
                    return this.LexXmlAttributeTextToken();
                case LexerMode.XmlCDataSectionText:
                    return this.LexXmlCDataSectionTextToken();
                case LexerMode.XmlCommentText:
                    return this.LexXmlCommentTextToken();
                case LexerMode.XmlProcessingInstructionText:
                    return this.LexXmlProcessingInstructionTextToken();
                case LexerMode.XmlCrefQuote:
                case LexerMode.XmlCrefDoubleQuote:
                    return this.LexXmlCrefOrNameToken();
                case LexerMode.XmlNameQuote:
                case LexerMode.XmlNameDoubleQuote:
                    // Same lexing as a cref attribute, just treat the identifiers a little differently.
                    return this.LexXmlCrefOrNameToken();
                case LexerMode.XmlCharacter:
                    return this.LexXmlCharacter();
            }

            Debug.Assert(false, "Unknown LexMode passed to Lexer.Lex");
            return this.LexSyntaxToken();
        }