C# CommonTokenStream.Consume方法代码示例

        private void UpdateQuickInfoContent(IQuickInfoSession session, SnapshotPoint triggerPoint)
        {
            /* use the experimental model to locate and process the expression */
            Stopwatch stopwatch = Stopwatch.StartNew();

            // lex the entire document
            var currentSnapshot = triggerPoint.Snapshot;
            var input = new SnapshotCharStream(currentSnapshot, new Span(0, currentSnapshot.Length));
            var unicodeInput = new JavaUnicodeStream(input);
            var lexer = new Java2Lexer(unicodeInput);
            var tokens = new CommonTokenStream(lexer);
            tokens.Fill();

            // locate the last token before the trigger point
            while (true)
            {
                IToken nextToken = tokens.LT(1);
                if (nextToken.Type == CharStreamConstants.EndOfFile)
                    break;

                if (nextToken.StartIndex > triggerPoint.Position)
                    break;

                tokens.Consume();
            }

            IToken triggerToken = tokens.LT(-1);
            if (triggerToken == null)
                return;

            switch (triggerToken.Type)
            {
            // symbol references
            case Java2Lexer.IDENTIFIER:
            case Java2Lexer.THIS:
            case Java2Lexer.SUPER:
            // primitive types
            case Java2Lexer.BOOLEAN:
            case Java2Lexer.CHAR:
            case Java2Lexer.BYTE:
            case Java2Lexer.SHORT:
            case Java2Lexer.INT:
            case Java2Lexer.LONG:
            case Java2Lexer.FLOAT:
            case Java2Lexer.DOUBLE:
            // literals
            case Java2Lexer.INTLITERAL:
            case Java2Lexer.LONGLITERAL:
            case Java2Lexer.FLOATLITERAL:
            case Java2Lexer.DOUBLELITERAL:
            case Java2Lexer.CHARLITERAL:
            case Java2Lexer.STRINGLITERAL:
            case Java2Lexer.TRUE:
            case Java2Lexer.FALSE:
            case Java2Lexer.NULL:
                break;

            default:
                return;
            }

            NetworkInterpreter interpreter = CreateNetworkInterpreter(tokens);

            while (interpreter.TryStepBackward())
            {
                if (interpreter.Contexts.Count == 0 || interpreter.Contexts.Count > 400)
                    break;

                if (interpreter.Contexts.All(context => context.BoundedStart))
                    break;
            }

            interpreter.Contexts.RemoveAll(i => !i.BoundedStart);
            interpreter.CombineBoundedStartContexts();

            IOutputWindowPane pane = Provider.OutputWindowService.TryGetPane(PredefinedOutputWindowPanes.TvlIntellisense);
            if (pane != null)
            {
                pane.WriteLine(string.Format("Located {0} QuickInfo expression(s) in {1}ms.", interpreter.Contexts.Count, stopwatch.ElapsedMilliseconds));
            }

            HashSet<string> intermediateResult = new HashSet<string>();
            HashSet<string> finalResult = new HashSet<string>();
            List<object> quickInfoContent = new List<object>();
            foreach (var context in interpreter.Contexts)
            {
                Span? span = null;

                foreach (var transition in context.Transitions)
                {
                    if (!transition.Transition.IsMatch)
                        continue;

                    IToken token = transition.Token;
                    Span tokenSpan = new Span(token.StartIndex, token.StopIndex - token.StartIndex + 1);
                    if (span == null)
                        span = tokenSpan;
                    else
                        span = Span.FromBounds(Math.Min(span.Value.Start, tokenSpan.Start), Math.Max(span.Value.End, tokenSpan.End));
                }
//.........这里部分代码省略.........

        public void AugmentQuickInfoSession(IQuickInfoSession session, IList<object> quickInfoContent, out ITrackingSpan applicableToSpan)
        {
            applicableToSpan = null;
            if (session == null || quickInfoContent == null)
                return;

            if (session.TextView.TextBuffer == this.TextBuffer)
            {
                ITextSnapshot currentSnapshot = this.TextBuffer.CurrentSnapshot;
                SnapshotPoint? triggerPoint = session.GetTriggerPoint(currentSnapshot);
                if (!triggerPoint.HasValue)
                    return;

                #region experimental
                /* use the experimental model to locate and process the expression */
                Stopwatch stopwatch = Stopwatch.StartNew();

                // lex the entire document
                var input = new SnapshotCharStream(currentSnapshot, new Span(0, currentSnapshot.Length));
                var lexer = new AlloyLexer(input);
                var tokens = new CommonTokenStream(lexer);
                tokens.Fill();

                // locate the last token before the trigger point
                while (true)
                {
                    IToken nextToken = tokens.LT(1);
                    if (nextToken.Type == CharStreamConstants.EndOfFile)
                        break;

                    if (nextToken.StartIndex > triggerPoint.Value.Position)
                        break;

                    tokens.Consume();
                }

                switch (tokens.LA(-1))
                {
                case AlloyLexer.IDENTIFIER:
                case AlloyLexer.KW_THIS:
                case AlloyLexer.KW_UNIV:
                case AlloyLexer.KW_IDEN:
                case AlloyLexer.KW_INT2:
                case AlloyLexer.KW_SEQINT:
                case AlloyLexer.INTEGER:
                    break;

                default:
                    return;
                }

                Network network = NetworkBuilder<AlloySimplifiedAtnBuilder>.GetOrBuildNetwork();

                RuleBinding memberSelectRule = network.GetRule(AlloySimplifiedAtnBuilder.RuleNames.BinOpExpr18);
#if DEBUG && false
                HashSet<Transition> memberSelectTransitions = new HashSet<Transition>(ObjectReferenceEqualityComparer<Transition>.Default);
                GetReachableTransitions(memberSelectRule, memberSelectTransitions);
#endif

                NetworkInterpreter interpreter = new NetworkInterpreter(network, tokens);

                interpreter.BoundaryRules.Add(memberSelectRule);
                //interpreter.BoundaryRules.Add(network.GetRule(AlloySimplifiedAtnBuilder.RuleNames.UnaryExpression));
                interpreter.BoundaryRules.Add(network.GetRule(AlloySimplifiedAtnBuilder.RuleNames.LetDecl));
                interpreter.BoundaryRules.Add(network.GetRule(AlloySimplifiedAtnBuilder.RuleNames.NameListName));
                interpreter.BoundaryRules.Add(network.GetRule(AlloySimplifiedAtnBuilder.RuleNames.Ref));
                interpreter.BoundaryRules.Add(network.GetRule(AlloySimplifiedAtnBuilder.RuleNames.Module));
                interpreter.BoundaryRules.Add(network.GetRule(AlloySimplifiedAtnBuilder.RuleNames.Open));
                interpreter.BoundaryRules.Add(network.GetRule(AlloySimplifiedAtnBuilder.RuleNames.FactDecl));
                interpreter.BoundaryRules.Add(network.GetRule(AlloySimplifiedAtnBuilder.RuleNames.AssertDecl));
                interpreter.BoundaryRules.Add(network.GetRule(AlloySimplifiedAtnBuilder.RuleNames.FunctionName));
                interpreter.BoundaryRules.Add(network.GetRule(AlloySimplifiedAtnBuilder.RuleNames.CmdDecl));
                interpreter.BoundaryRules.Add(network.GetRule(AlloySimplifiedAtnBuilder.RuleNames.Typescope));
                interpreter.BoundaryRules.Add(network.GetRule(AlloySimplifiedAtnBuilder.RuleNames.EnumDecl));
                interpreter.BoundaryRules.Add(network.GetRule(AlloySimplifiedAtnBuilder.RuleNames.ElseClause));

                interpreter.ExcludedStartRules.Add(network.GetRule(AlloySimplifiedAtnBuilder.RuleNames.CallArguments));

                while (interpreter.TryStepBackward())
                {
                    if (interpreter.Contexts.Count == 0)
                        break;

                    /* we want all traces to start outside the binOpExpr18 rule, which means all
                     * traces with a transition reachable from binOpExpr18 should contain a push
                     * transition with binOpExpr18's start state as its target.
                     */
                    if (interpreter.Contexts.All(context => context.BoundedStart))
                    {
                        break;
                    }
                }

                HashSet<InterpretTrace> contexts = new HashSet<InterpretTrace>(BoundedStartInterpretTraceEqualityComparer.Default);
                if (interpreter.Contexts.Count > 0)
                {
                    contexts.UnionWith(interpreter.Contexts);
                }
                else
                {
//.........这里部分代码省略.........

        public void TestOffChannel()
        {
            ITokenSource lexer = // simulate input " x =34  ;\n"
                new TestOffChannelTokenSource();

            CommonTokenStream tokens = new CommonTokenStream(lexer);

            Assert.AreEqual("x", tokens.LT(1).Text); // must skip first off channel token
            tokens.Consume();
            Assert.AreEqual("=", tokens.LT(1).Text);
            Assert.AreEqual("x", tokens.LT(-1).Text);

            tokens.Consume();
            Assert.AreEqual("34", tokens.LT(1).Text);
            Assert.AreEqual("=", tokens.LT(-1).Text);

            tokens.Consume();
            Assert.AreEqual(";", tokens.LT(1).Text);
            Assert.AreEqual("34", tokens.LT(-1).Text);

            tokens.Consume();
            Assert.AreEqual(CharStreamConstants.EndOfFile, tokens.LA(1));
            Assert.AreEqual(";", tokens.LT(-1).Text);

            Assert.AreEqual("34", tokens.LT(-2).Text);
            Assert.AreEqual("=", tokens.LT(-3).Text);
            Assert.AreEqual("x", tokens.LT(-4).Text);
        }

        protected override void ReParseImpl()
        {
            // lex the entire document to get the set of identifiers we'll need to process
            ITextSnapshot snapshot = TextBuffer.CurrentSnapshot;
            var input = new SnapshotCharStream(snapshot, new Span(0, snapshot.Length));
            var lexer = new AlloyLexer(input);
            var tokens = new CommonTokenStream(lexer);
            tokens.Fill();

            /* Want to collect information from the following:
             *  - module (name)
             * Want to provide navigation info for the following types:
             *  - sig
             *  - enum
             * Want to provide navigation info for the following members:
             *  - decl (within a sigBody)
             *  - fun
             *  - pred
             *  - nameList (within an enumBody)
             * Eventually should consider the following:
             *  - cmdDecl
             *  - fact
             *  - assert
             */

            List<IToken> navigationKeywords = new List<IToken>();
            while (tokens.LA(1) != CharStreamConstants.EndOfFile)
            {
                switch (tokens.LA(1))
                {
                case AlloyLexer.KW_MODULE:
                case AlloyLexer.KW_SIG:
                case AlloyLexer.KW_ENUM:
                case AlloyLexer.KW_FUN:
                case AlloyLexer.KW_PRED:
                //case AlloyLexer.KW_ASSERT:
                //case AlloyLexer.KW_FACT:
                    navigationKeywords.Add(tokens.LT(1));
                    break;

                case CharStreamConstants.EndOfFile:
                    goto doneLexing;

                default:
                    break;
                }

                tokens.Consume();
            }

        doneLexing:

            List<IEditorNavigationTarget> navigationTargets = new List<IEditorNavigationTarget>();
            AstParserRuleReturnScope<CommonTree, IToken> moduleTree = null;
            CommonTreeAdaptor treeAdaptor = new CommonTreeAdaptor();

            foreach (var token in navigationKeywords)
            {
                tokens.Seek(token.TokenIndex);
                tokens.Consume();

                NetworkInterpreter interpreter = CreateNetworkInterpreter(tokens);
                while (interpreter.TryStepBackward())
                {
                    if (interpreter.Contexts.Count == 0)
                        break;

                    if (interpreter.Contexts.All(context => context.BoundedStart))
                        break;
                }

                interpreter.CombineBoundedStartContexts();

#if false // since we're using the AlloyParser, I don't think we need this.
                while (interpreter.TryStepForward())
                {
                    if (interpreter.Contexts.Count == 0)
                        break;

                    if (interpreter.Contexts.All(context => context.BoundedEnd))
                        break;
                }
#endif

                foreach (var context in interpreter.Contexts)
                {
                    switch (token.Type)
                    {
                    case AlloyLexer.KW_MODULE:
                        {
                            InterpretTraceTransition firstMatch = context.Transitions.FirstOrDefault(i => i.TokenIndex != null);
                            if (firstMatch == null)
                                continue;

                            tokens.Seek(firstMatch.TokenIndex.Value);
                            AlloyParser parser = new AlloyParser(tokens);
                            AstParserRuleReturnScope<CommonTree, IToken> result = parser.module();
                            if (result == null || parser.NumberOfSyntaxErrors > 0)
                                continue;

//.........这里部分代码省略.........

        public void AugmentQuickInfoSession(IQuickInfoSession session, IList<object> quickInfoContent, out ITrackingSpan applicableToSpan)
        {
            applicableToSpan = null;
            if (session == null || quickInfoContent == null)
                return;

            if (session.TextView.TextBuffer == this.TextBuffer)
            {
                ITextSnapshot currentSnapshot = this.TextBuffer.CurrentSnapshot;
                SnapshotPoint? triggerPoint = session.GetTriggerPoint(currentSnapshot);
                if (!triggerPoint.HasValue)
                    return;

                #region experimental
                /* use the experimental model to locate and process the expression */
                Stopwatch stopwatch = Stopwatch.StartNew();

                // lex the entire document
                var input = new SnapshotCharStream(currentSnapshot, new Span(0, currentSnapshot.Length));
                var lexer = new GoLexer(input);
                var tokenSource = new GoSemicolonInsertionTokenSource(lexer);
                var tokens = new CommonTokenStream(tokenSource);
                tokens.Fill();

                // locate the last token before the trigger point
                while (true)
                {
                    IToken nextToken = tokens.LT(1);
                    if (nextToken.Type == CharStreamConstants.EndOfFile)
                        break;

                    if (nextToken.StartIndex > triggerPoint.Value.Position)
                        break;

                    tokens.Consume();
                }

                switch (tokens.LA(-1))
                {
                case GoLexer.IDENTIFIER:
                //case GoLexer.KW_THIS:
                //case GoLexer.KW_UNIV:
                //case GoLexer.KW_IDEN:
                //case GoLexer.KW_INT2:
                //case GoLexer.KW_SEQINT:
                    break;

                default:
                    return;
                }

                Network network = NetworkBuilder<GoSimplifiedAtnBuilder>.GetOrBuildNetwork();

                RuleBinding memberSelectRule = network.GetRule(GoSimplifiedAtnBuilder.RuleNames.PrimaryExpr);
#if false
                HashSet<Transition> memberSelectTransitions = new HashSet<Transition>();
                GetReachableTransitions(memberSelectRule, memberSelectTransitions);
#endif

                NetworkInterpreter interpreter = new NetworkInterpreter(network, tokens);

                interpreter.BoundaryRules.Add(memberSelectRule);
                interpreter.BoundaryRules.Add(network.GetRule(GoSimplifiedAtnBuilder.RuleNames.Label));
                interpreter.BoundaryRules.Add(network.GetRule(GoSimplifiedAtnBuilder.RuleNames.TypeSwitchGuard));
                interpreter.BoundaryRules.Add(network.GetRule(GoSimplifiedAtnBuilder.RuleNames.FieldName));
                interpreter.BoundaryRules.Add(network.GetRule(GoSimplifiedAtnBuilder.RuleNames.Receiver));
                interpreter.BoundaryRules.Add(network.GetRule(GoSimplifiedAtnBuilder.RuleNames.FunctionDecl));
                interpreter.BoundaryRules.Add(network.GetRule(GoSimplifiedAtnBuilder.RuleNames.BaseTypeName));
                interpreter.BoundaryRules.Add(network.GetRule(GoSimplifiedAtnBuilder.RuleNames.TypeSpec));
                interpreter.BoundaryRules.Add(network.GetRule(GoSimplifiedAtnBuilder.RuleNames.IdentifierList));
                interpreter.BoundaryRules.Add(network.GetRule(GoSimplifiedAtnBuilder.RuleNames.MethodName));
                interpreter.BoundaryRules.Add(network.GetRule(GoSimplifiedAtnBuilder.RuleNames.ParameterDecl));
                interpreter.BoundaryRules.Add(network.GetRule(GoSimplifiedAtnBuilder.RuleNames.FieldIdentifierList));
                interpreter.BoundaryRules.Add(network.GetRule(GoSimplifiedAtnBuilder.RuleNames.PackageName));
                interpreter.BoundaryRules.Add(network.GetRule(GoSimplifiedAtnBuilder.RuleNames.TypeName));

                interpreter.ExcludedStartRules.Add(network.GetRule(GoSimplifiedAtnBuilder.RuleNames.Block));

                while (interpreter.TryStepBackward())
                {
                    if (interpreter.Contexts.Count == 0)
                        break;

                    /* we want all traces to start outside the binOpExpr18 rule, which means all
                     * traces with a transition reachable from binOpExpr18 should contain a push
                     * transition with binOpExpr18's start state as its target.
                     */
                    if (interpreter.Contexts.All(context => context.BoundedStart))
                    {
                        break;
                    }
                }

                interpreter.CombineBoundedStartContexts();

                IOutputWindowPane pane = Provider.OutputWindowService.TryGetPane(PredefinedOutputWindowPanes.TvlIntellisense);
                if (pane != null)
                {
                    pane.WriteLine(string.Format("Located {0} QuickInfo expression(s) in {1}ms.", interpreter.Contexts.Count, stopwatch.ElapsedMilliseconds));
                }
//.........这里部分代码省略.........