C# TokenStream.Advance方法代码示例

        public void AllowsRepeatableTraversalWhileTraversingUnderlyingEnumeratorItemsAtMostOnce()
        {
            var tokens = new TokenStream(Tokens());

            tokens.Current.ShouldBe(upper, "ABC", 1, 1);
            tokens.Advance().Current.ShouldBe(lower, "def", 1, 4);
            tokens.Advance().Advance().Current.ShouldBe(upper, "GHI", 1, 7);
            tokens.Advance().Advance().Advance().Current.ShouldBe(TokenKind.EndOfInput, "", 1, 10);

            tokens.Advance().ShouldBeSameAs(tokens.Advance());
        }

        private static Ast.New ParseNew(
			TokenStream Stream,
			ParseContext Context)
        {
            Stream.Advance();
            if (Stream.Next().Type != TokenType.Identifier)
                throw new CompileError("[01D] Expected identifier", Stream);
            var r = new Ast.New(Stream.Next());
            r.Typename = Stream.Next().Value.ToUpper();
            Stream.Advance();
            if (Stream.Next().Type == TokenType.OpenBrace)
                r.Initializers = ParseInitializers(Stream, Context);
            return r;
        }

        public static List<Ast.Node> ParseStaticInvokation(
			TokenStream Stream,
			ParseContext Context)
        {
            if (Stream.Next().Type != TokenType.OpenBracket) throw new CompileError("[017] Expected [", Stream);
            Stream.Advance();

             var parameters = new List<Ast.Node>();
             while (true)
             {
                 if (IsEndOfStatement(Stream)) throw new CompileError("[018] Expected ]", Stream);
                 if (Stream.Next().Type == TokenType.CloseBracket)
                 {
                     Stream.Advance();
                     return parameters;
                 }
                 parameters.Add(ParseTerm(Stream, Context));
             }
        }

        public void Style(TokenStream Stream)
        {
            this.Stream = Stream;
            this.FoldLevel = 1;

            while (!Stream.AtEnd())
            {
                if (Stream.Next().Value.ToUpper() == "MACRO"
                    || Stream.Next().Value.ToUpper() == "FUNCTION"
                    || Stream.Next().Value.ToUpper() == "TEST")
                    ParseMacroDeclaration();
                else if (Stream.Next().Value.ToUpper() == "RULE")
                    ParseRuleDeclaration();
                else if (Stream.Next().Value.ToUpper() == "TYPE")
                    ParseTypeDeclaration();
                else if (Stream.Next().Value.ToUpper() == "GLOBAL")
                    ParseGlobalDeclaration();
                else if (Stream.Next().Value.ToUpper() == "INCLUDE")
                {
                    Style(TokenStyle.Keyword);
                    Expect(TokenType.String);
                }
                else if (Stream.Next().Value.ToUpper() == "DEFAULT")
                {
                    Style(TokenStyle.Keyword);
                    Expect(TokenType.Identifier, t => t.Value.ToUpper() == "OF");
                    ParseRuleDeclaration();
                }
                else
                    Stream.Advance();
            }
        }

        public void ProvidesEndOfInputTokenAfterEnumeratorIsExhausted()
        {
            var tokens = new TokenStream(OneToken());
            var end = tokens.Advance();

            end.Current.ShouldBe(TokenKind.EndOfInput, "", 1, 4);
            end.Advance().ShouldBeSameAs(end);
        }

 public void AdvancesToTheNextToken()
 {
     var tokens = new TokenStream(Tokens());
     tokens.Advance().Current.ShouldBe(lower, "def", 1, 4);
 }

        public void ProvidesPositionOfCurrentToken()
        {
            var tokens = new TokenStream(Tokens());

            tokens.Position.Line.ShouldBe(1);
            tokens.Position.Column.ShouldBe(1);

            tokens.Advance().Position.Line.ShouldBe(1);
            tokens.Advance().Position.Column.ShouldBe(4);

            tokens.Advance().Advance().Position.Line.ShouldBe(1);
            tokens.Advance().Advance().Position.Column.ShouldBe(7);

            tokens.Advance().Advance().Advance().Position.Line.ShouldBe(1);
            tokens.Advance().Advance().Advance().Position.Column.ShouldBe(10);
        }

        private static Ast.Statement ParseStatement(
			TokenStream Stream,
			ParseContext Context)
        {
            Ast.Statement r = null;

            var firstToken = Stream.Next().Value.ToUpper();

            if (firstToken == "LET")
            {
                r = ParseLetStatement(Stream, Context);
            }
            else if (firstToken == "IF")
            {
                r = ParseIfStatement(Stream, Context);
            }
            else if (firstToken == "RETURN")
            {
                r = ParseReturnStatement(Stream, Context);
            }
            else if (firstToken == "VAR" || firstToken == "VARIABLE")
            {
                r = ParseLocalDeclaration(Stream, Context);
            }
            else
            {
                //If it's not any special statement, it must be a function call.
                var parameters = ParseStaticInvokationStatement(Stream, Context);
                var control = Context.FindControl(parameters);
                if (control != null)
                {
                    Ast.Node childBlock = null;
                    if (control.BlockType == ControlBlockType.RequiredBlock)
                    {
                        if (Stream.Next().Type == TokenType.OpenBrace)
                            childBlock = ParseBlock(Stream, Context);
                        else
                            throw new CompileError("[00C] Expected {", Stream);
                    }
                    else
                    {
                        if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[00D] Expected ;", Stream);
                        Stream.Advance();
                    }
                    r = new Ast.ControlInvokation(parameters[0].Source, control, parameters, childBlock);
                }
                else
                {
                    r = new Ast.StaticInvokation(parameters[0].Source, parameters);
                    if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[00E] Expected ;", Stream);
                    Stream.Advance();
                }
            }

            return r;
        }

        internal static Type ParseTypeDeclaration(TokenStream Stream, ParseContext Context)
        {
            Stream.Advance();

            if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[02C] Expected identifier", Stream);
            var typename = Stream.Next().Value.ToUpper();
            String superTypename;

            Stream.Advance();
            if (Stream.Next().Type == TokenType.Colon)
            {
                Stream.Advance();
                if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("Expected identifier", Stream);
                superTypename = Stream.Next().Value.ToUpper();
                Stream.Advance();
            }
            else
                superTypename = "GENERIC";

            var r = new Type(superTypename) { Name = typename };

            if (Stream.Next().Type != TokenType.OpenBrace) throw new CompileError("[02D] Expected {", Stream);
            Stream.Advance();

            while (Stream.Next().Type != TokenType.CloseBrace)
            {
                if (Stream.Next().Value.ToUpper() == "VAR" || Stream.Next().Value.ToUpper() == "VARIABLE")
                    r.Members.Add(ParseMemberDeclaration(Stream, Context));
                else
                    throw new CompileError("[02E] Expected var", Stream);
            }

            Stream.Advance();

            if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark)
            {
                Stream.Advance();
                if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream);
                r.Documentation = Stream.Next().Value;
                Stream.Advance();
            }

            return r;
        }

        internal static Declaration ParseRuleDeclaration(TokenStream Stream, ParseContext Context)
        {
            if (Stream.AtEnd())
                throw new CompileError("[02F] Impossible error: ParseRuleDeclaration entered at end of stream.", Stream);

            try
            {
                var r = new Declaration();
                r.ReturnTypeName = "RULE-RESULT";

                if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[030] Expected identifier", Stream.Next());

                r.Type = DeclarationType.Rule;
                Stream.Advance();

                r.Terms = ParseMacroDeclarationHeader(Stream, DeclarationHeaderTerminatorType.OpenBraceOrWhen);
                foreach (var t in r.Terms)
                    if (t.Type == DeclarationTermType.Operator)
                        throw new CompileError("Rule headers cannot contain operators", Stream);

                if (Stream.Next().Type == TokenType.Colon)
                {
                    Stream.Advance();
                    if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[02A] Expected identifier", Stream);
                    r.ReturnTypeName = Stream.Next().Value.ToUpper();
                    Stream.Advance();
                }

                if (Stream.Next().Value.ToUpper() == "WHEN")
                {
                    Stream.Advance();
                    r.WhenClause = new WhenClause(ParseExpression(Stream, Context, (stream) =>
                        {
                            if (stream.Next().Type == TokenType.OpenBrace) return true;
                            if (stream.Next().Type == TokenType.Identifier &&
                                (stream.Next().Value.ToUpper() == "WITH" ||
                                stream.Next().Value.ToUpper() == "ORDER")) return true;
                            return false;
                        }));
                }

                r.OrderOperator = OrderOperator.NONE;

                if (Stream.Next().Value.ToUpper() == "ORDER")
                {
                    Stream.Advance();
                    var temp = OrderOperator.NONE;
                    if (!Enum.TryParse(Stream.Next().Value.ToUpper(), out temp))
                        throw new CompileError("Expected FIRST or LAST", Stream);
                    if (temp != OrderOperator.FIRST && temp != OrderOperator.LAST)
                        throw new CompileError("Expected FIRST or LAST", Stream);
                    Stream.Advance();

                    r.OrderOperator |= temp;
                }

                if (Stream.Next().Value.ToUpper() == "WITH")
                {
                    Stream.Advance();
                    var temp = OrderOperator.NONE;
                    if (!Enum.TryParse(Stream.Next().Value.ToUpper(), out temp))
                        throw new CompileError("Expected HIGH or LOW", Stream);
                    if (temp != OrderOperator.HIGH && temp != OrderOperator.LOW)
                        throw new CompileError("Expected HIGH or LOW", Stream);
                    Stream.Advance();
                    if (Stream.Next().Value.ToUpper() != "PRIORITY")
                        throw new CompileError("Expected PRIORITY", Stream);
                    Stream.Advance();

                    r.OrderOperator |= temp;
                }

                if (Stream.Next().Type == TokenType.OpenBrace)
                {
                    r.Body = new LambdaBlock(ParseBlock(Stream, Context));

                }
                else
                {	throw new CompileError("[031] Expected block", Stream);
                }

                if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark)
                {
                    Stream.Advance();
                    if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream);
                    r.Documentation = Stream.Next().Value;
                    Stream.Advance();
                }

                return r;
            }
            catch (CompileError ce)
            {
                throw ce;
            }
            catch (Exception e)
            {
                throw new CompileError(e.Message + e.StackTrace, Stream);
            }
        }

        internal static Variable ParseMemberDeclaration(TokenStream Stream, ParseContext Context)
        {
            if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[032] Expected identifier", Stream.Next());
            Stream.Advance();

            var r = new Variable();
            r.StorageMethod = VariableStorageMethod.Member;

            if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[033] Expected identifier", Stream.Next());
            r.Name = Stream.Next().Value.ToUpper();

            Stream.Advance();

            if (Stream.Next().Type == TokenType.Colon)
            {
                Stream.Advance();
                if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[034] Expected identifier", Stream.Next());
                r.DeclaredTypeName = Stream.Next().Value.ToUpper();
                Stream.Advance();
            }

            if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark)
            {
                Stream.Advance();
                if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream);
                r.Documentation = Stream.Next().Value;
                Stream.Advance();
            }

            if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[035] Expected ;", Stream.Next());
            Stream.Advance();
            return r;
        }

        internal static Declaration ParseMacroDeclaration(TokenStream Stream, ParseContext Context)
        {
            if (Stream.AtEnd())
                throw new CompileError("[028] Impossible error: ParseDeclaration entered at end of stream.", Stream);

            try
            {
                var r = new Declaration();
                r.ReturnTypeName = "VOID";

                if (Stream.Next().Type != TokenType.Identifier)
                    throw new CompileError("[029] Expected identifier", Stream.Next());

                r.Type = DeclarationType.Macro;
                Stream.Advance();

                r.Terms = ParseMacroDeclarationHeader(Stream, DeclarationHeaderTerminatorType.OpenBrace);
                foreach (var t in r.Terms)
                    if (t.Type == DeclarationTermType.Operator)
                        r.DefinesOperator = true;

                if (r.DefinesOperator)
                {
                    if (r.Terms.Count != 3 ||
                        r.Terms[0].Type != DeclarationTermType.Term ||
                        r.Terms[1].Type != DeclarationTermType.Operator ||
                        r.Terms[2].Type != DeclarationTermType.Term)
                        throw new CompileError("Operator macros must be of the form 'term op term'", Stream);

                    r.Terms[1].Type = DeclarationTermType.Keyword;
                }

                if (!Stream.AtEnd() && Stream.Next().Type == TokenType.Colon)
                {
                    Stream.Advance();
                    if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[02A] Expected identifier", Stream);
                    r.ReturnTypeName = Stream.Next().Value.ToUpper();
                    Stream.Advance();
                }

                if (r.DefinesOperator && r.ReturnTypeName == "VOID")
                    throw new CompileError("Operator macros must return a value.", Stream);

                if (!Stream.AtEnd() && Stream.Next().Type == TokenType.OpenBrace)
                {
                    r.Body = new LambdaBlock(ParseBlock(Stream, Context));
                }
                else
                    throw new CompileError("[02B] Expected block", Stream);

                if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark)
                {
                    Stream.Advance();
                    if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream);
                    r.Documentation = Stream.Next().Value;
                    Stream.Advance();
                }

                return r;
            }
            catch (CompileError ce)
            {
                throw ce;
            }
            catch (Exception e)
            {
                throw new CompileError(e.Message + e.StackTrace, Stream);
            }
        }

        internal static Variable ParseGlobalDeclaration(TokenStream Stream, ParseContext Context)
        {
            if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[032] Expected identifier", Stream.Next());
            Stream.Advance();

            var r = new Variable();
            r.StorageMethod = VariableStorageMethod.Member;

            if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[033] Expected identifier", Stream.Next());
            var start = Stream.Next();
            r.Name = Stream.Next().Value.ToUpper();

            Stream.Advance();

            if (Stream.Next().Type == TokenType.Colon)
            {
                Stream.Advance();
                if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[034] Expected identifier", Stream.Next());
                r.DeclaredTypeName = Stream.Next().Value.ToUpper();
                Stream.Advance();
            }

            if (Stream.Next().Type == TokenType.Operator && Stream.Next().Value == "=")
            {
                Stream.Advance();
                var initialValue = ParseExpression(Stream, Context, TokenType.Semicolon);
                var initializer = new Ast.Let(start, new Ast.Identifier(start), initialValue);
                Context.Initialization.Add(initializer);
            }

            if (!Stream.AtEnd() && Stream.Next().Type == TokenType.QuestionMark)
            {
                Stream.Advance();
                if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected documentation", Stream);
                r.Documentation = Stream.Next().Value;
                Stream.Advance();
            }

            if (Stream.Next().Type != TokenType.Semicolon) throw new CompileError("[035] Expected ;", Stream.Next());
            Stream.Advance();
            return r;
        }

        public static List<Declaration> Build(
			TokenStream Stream, 
			ParseContext Context,
			Func<String,ErrorStrategy> OnError,
			bool PrepareInitializer = true)
        {
            var r = new List<Declaration>();
            while (!Stream.AtEnd())
            {
                try
                {
                    if (Stream.Next().Type != TokenType.Identifier) throw new CompileError("[036] Expected identifier", Stream);
                    if (Stream.Next().Value.ToUpper() == "MACRO" || Stream.Next().Value.ToUpper() == "FUNCTION")
                    {
                        var declaration = ParseMacroDeclaration(Stream, Context);
                        declaration.OwnerContextID = Context.ID;
                        Context.PendingEmission.Add(declaration);
                    }
                    else if (Stream.Next().Value.ToUpper() == "TEST")
                    {
                        var declaration = ParseMacroDeclaration(Stream, Context);
                        declaration.Type = DeclarationType.Test;
                        declaration.OwnerContextID = Context.ID;
                        r.Add(declaration);
                        Context.PendingEmission.Add(declaration);
                    }
                    else if (Stream.Next().Value.ToUpper() == "RULE")
                    {
                        var declaration = ParseRuleDeclaration(Stream, Context);
                        declaration.OwnerContextID = Context.ID;
                        var rulebook = Context.Rules.FindMatchingRulebook(declaration.Terms);
                        if (rulebook == null)
                        {
                            rulebook = new Rulebook { DeclarationTerms = new List<DeclarationTerm>(
                                declaration.Terms.Select(t => t.GenericClone())) };
                            rulebook.ResultTypeName = declaration.ReturnTypeName;
                            Context.Rules.Rulebooks.Add(rulebook);
                        }
                        //if (Declaration.AreTermTypesCompatible(rulebook.DeclarationTerms, declaration.Terms) == false)
                        //	throw new CompileError("[037] Term types are not compatible with existing rulebook", Stream);
                        if (declaration.ReturnTypeName.ToUpper() != rulebook.ResultTypeName.ToUpper())
                            throw new CompileError("Rule return type not compatible with existing rulebook", Stream);
                        rulebook.Rules.Add(declaration);
                        Context.PendingEmission.Add(declaration);
                    }
                    else if (Stream.Next().Value.ToUpper() == "TYPE")
                    {
                        var type = ParseTypeDeclaration(Stream, Context);
                        if (Context.ActiveScope.FindType(type.Name) != null) throw new CompileError("[038] Type already defined", Stream);
                        Context.ActiveScope.Types.Add(type);
                    }
                    else if (Stream.Next().Value.ToUpper() == "GLOBAL")
                    {
                        var variable = ParseGlobalDeclaration(Stream, Context);
                        variable.StorageMethod = VariableStorageMethod.Static;
                        Context.ActiveScope.Variables.Add(variable);
                    }
                    else if (Stream.Next().Value.ToUpper() == "INCLUDE")
                    {
                        Stream.Advance();
                        if (Stream.Next().Type != TokenType.String) throw new CompileError("Expected string", Stream);
                        var filename = Stream.Next().Value;
                        Stream.Advance();
                        if (Context.FileLoader == null) throw new CompileError("Inclusion not enabled", Stream);
                        var file = Context.FileLoader.LoadFile(filename, Stream.CurrentFile);
                        var newStream = new TokenStream(new StringIterator(file.Data), Context);
                        newStream.CurrentFile = file;
                        r.AddRange(Build(newStream, Context, OnError, false));
                    }
                    else if (Stream.Next().Value.ToUpper() == "DEFAULT")
                    {
                        Stream.Advance();
                        if (Stream.Next().Value.ToUpper() != "OF") throw new CompileError("Expected 'OF'", Stream);
                        Stream.Advance();
                        if (Stream.Next().Value.ToUpper() != "RULE") throw new CompileError("Expected 'RULE'", Stream);
                        var declaration = ParseRuleDeclaration(Stream, Context);
                        foreach (var term in declaration.Terms)
                            if (!String.IsNullOrEmpty(term.DeclaredTypeName) && term.DeclaredTypeName != "GENERIC")
                                throw new CompileError("Don't declare types for default rules.");
                        declaration.OwnerContextID = Context.ID;
                        var rulebook = Context.Rules.FindMatchingRulebook(declaration.Terms);
                        if (rulebook == null)
                        {
                            rulebook = new Rulebook { DeclarationTerms = declaration.Terms };
                            rulebook.ResultTypeName = declaration.ReturnTypeName;
                            Context.Rules.Rulebooks.Add(rulebook);
                        }
                        //if (Declaration.AreTermTypesCompatible(rulebook.DeclarationTerms, declaration.Terms) == false)
                        //	throw new CompileError("[037] Term types are not compatible with existing rulebook", Stream);
                        if (declaration.ReturnTypeName.ToUpper() != rulebook.ResultTypeName.ToUpper())
                            throw new CompileError("Rule return type not compatible with existing rulebook", Stream);
                        if (rulebook.DefaultValue != null)
                            throw new CompileError("Rulebook already has a default value", Stream);
                        rulebook.DefaultValue = declaration;
                        Context.PendingEmission.Add(declaration);
                    }
                    else
                        throw new CompileError("[039] Unknown declaration type", Stream);
                }
                catch (Exception e)
//.........这里部分代码省略.........

        internal static DeclarationTerm ParseDeclarationTerm(TokenStream Stream)
        {
            DeclarationTerm r = null;
            var start = Stream.Next();

            if (Stream.Next().Type == TokenType.Identifier)
            {
                r = new DeclarationTerm
                {
                    Name = Stream.Next().Value.ToUpper(),
                    Type = DeclarationTermType.Keyword,
                    RepetitionType = DeclarationTermRepetitionType.Once
                };
                Stream.Advance();
            }
            else if (Stream.Next().Type == TokenType.Operator)
            {
                r = new DeclarationTerm
                {
                    Name = Stream.Next().Value.ToUpper(),
                    Type = DeclarationTermType.Operator,
                    RepetitionType = DeclarationTermRepetitionType.Once
                };
                Stream.Advance();
            }
            else if (Stream.Next().Type == TokenType.OpenParen)
            {
                Stream.Advance();
                if (Stream.Next().Type != TokenType.Identifier)
                    throw new CompileError("[021] Expected identifier", start);
                r = new DeclarationTerm
                {
                    Name = Stream.Next().Value.ToUpper(),
                    Type = DeclarationTermType.Term,
                    RepetitionType = DeclarationTermRepetitionType.Once
                };
                Stream.Advance();
                if (Stream.Next().Type == TokenType.Colon)
                {
                    Stream.Advance();
                    if (Stream.Next().Type != TokenType.Identifier)
                        throw new CompileError("[022] Expected identifier", start);
                    var declaredType = Stream.Next().Value;
                    r.DeclaredTypeName = declaredType.ToUpper();
                    Stream.Advance();
                }
                if (Stream.Next().Type != TokenType.CloseParen)
                    throw new CompileError("[023] Expected )", start);
                Stream.Advance();
            }
            else
                throw new CompileError("[025] Illegal token in declaration header", start);

            if (!Stream.AtEnd())
            {
                if (Stream.Next().Type == TokenType.QuestionMark)
                {
                    var marker = Stream.Next();
                    var repetitionMarker = Stream.Next().Value;
                    Stream.Advance();
                    if (repetitionMarker == "?")
                    {
                        if (r.Type != DeclarationTermType.Keyword)
                            throw new CompileError("[026] Only keywords can be optional in a declaration header", Stream);
                        r.RepetitionType = DeclarationTermRepetitionType.Optional;
                    }
                        //Left over from when terms could be repeated.
                    //else if (repetitionMarker == "+")
                    //    r.RepetitionType = DeclarationTermRepetitionType.OneOrMany;
                    //else if (repetitionMarker == "*")
                    //    r.RepetitionType = DeclarationTermRepetitionType.NoneOrMany;
                    else
                        throw new CompileError("[027] Unrecognized repetition marker on declaration term", marker);
                }
            }

            return r;
        }