C# ILexer类代码示例

		protected override ILexer<Token> PrepareLexer(ILexer<Token> lexer, ICharSource file, int position)
		{
			if (lexer == null)
				return new EcsLexer(file, "?", MessageSink.Trace, position);
			((EcsLexer)lexer).Reset(file, "?", position);
			return lexer;
		}

        public MoshParser(ILexer lexer)
        {
            m_startRuleToken = new Token { TokenType = TokenType.NEW_RULE };
            m_tree = new Node<string>();
            m_lexer = lexer;
            m_consumed = new Stack<Token>();
            m_err = new Queue<Error>();

            // Setup rule pre-calling conditions
            m_rulePreHook = name =>
                                {
                                    if (m_tree.Value == null)
                                    {
                                        m_tree.Value = name;
                                        return m_tree;
                                    }

                                    var tempNode = m_tree;
                                    var newTopNode = m_tree.AddChild(name);
                                    m_tree = newTopNode;
                                    return tempNode;
                                };

            // Setup rule post-calling conditions
            m_rulePostHook = node => m_tree = node;
        }

        public IEnumerable<ILexer> GetMatches(ILexer lexer, string ruleName)
        {
            var wrapRules = new List<GrammarRule>();
            var results = new List<ILexer>();

            foreach (var rule in _table.Get(ruleName))
            {
                if (rule.Elements[0] is GrammarRuleElementEmbed && ((GrammarRuleElementEmbed) rule.Elements[0]).RuleName == ruleName)
                {
                    if (rule.Elements.Length > 1)
                        wrapRules.Add(rule);
                }
                else
                {
                    foreach (var match in GetMatches(lexer, rule))
                    {
                        yield return match;
                        results.Add(match);
                    }
                }
            }

            foreach (var start in results)
                foreach (var match in MatchesRecursive(start, wrapRules))
                    yield return match;
        }

 private static void MainLoop(ILexer lexer, IParser parser)
 {
     // top ::= definition | external | expression | ';'
     while (true)
     {
         Console.Write("ready> ");
         switch (lexer.CurrentToken)
         {
         case (int)Token.EOF:
             return;
         case ';':
             lexer.GetNextToken();
             break;
         case (int)Token.DEF:
             parser.HandleDefinition();
             break;
         case (int)Token.EXTERN:
             parser.HandleExtern();
             break;
         default:
             parser.HandleTopLevelExpression();
             break;
         }
     }
 }

 public SpimiIndexer(ILexer lexer)
 {
     this.lexer = lexer;
     this.blockReader = new SpimiBlockReader();
     this.blockWriter = new SpimiBlockWriter();
     this.fileIndexWriter = new FileIndexWriter();
 }

		internal T4Parser([NotNull] T4Environment t4Environment, [NotNull] DirectiveInfoManager directiveInfoManager, [NotNull] ILexer lexer,
			[CanBeNull] IPsiSourceFile sourceFile) {
			_t4Environment = t4Environment;
			_directiveInfoManager = directiveInfoManager;
			_lexer = lexer;
			_sourceFile = sourceFile;
		}

    public static void Run(TreeElement node, ILexer lexer, ITokenOffsetProvider offsetProvider, bool trimTokens, SeldomInterruptChecker interruptChecker)
    {
      Assertion.Assert(node.parent == null, "node.parent == null");

      var root = node as CompositeElement;
      if (root == null)
      {
        return;
      }

      var inserter = new PsiMissingTokensInserter(lexer, offsetProvider, interruptChecker);
      lexer.Start();

      if (trimTokens)
      {
        using (var container = new DummyContainer(root))
        {
          inserter.Run(container);
        }
      }
      else
      {
        var terminator = new EofToken(lexer.Buffer);
        root.AppendNewChild(terminator);
        inserter.Run(root);
        root.DeleteChildRange(terminator, terminator);
      }
    }

 public PsiParser(ILexer lexer)
 {
   myCheckForInterrupt = new SeldomInterruptChecker();
   myOriginalLexer = lexer;
   myLexer = new FilteringPsiLexer(lexer);
   myLexer.Start();
 }

 public NTriplesParser(ILexer lexer, CommonIdentifierIntern commonIdentifierIntern)
 {
     this.commonIdentifierIntern = commonIdentifierIntern;
     this.originalLexer = lexer;
     this.myCheckForInterrupt = new SeldomInterruptChecker();
     this.setLexer(new NTriplesFilteringLexer(lexer));
 }

 /// <inheritdoc />
 public ILexer<Repetition> Create(ILexer lexer)
 {
     if (lexer == null)
     {
         throw new ArgumentNullException(nameof(lexer));
     }
     return new OptionLexer(lexer);
 }

        public Parser(ILexer lexer)
        {
            this.lexer = lexer;

            // prime the lookahead
            this.currentToken = lexer.GetToken ();
            this.nextToken = lexer.GetToken ();
        }

 public SpimiIndexer(ILexer lexer, IParser parser, Stream indexStream, Stream metadata)
 {
     this.lexer = lexer;
     this.parser = parser;
     this.termIndexBlockWriter = new SpimiBlockWriter();
     this.indexStream = indexStream;
     this.metadataWriter = new CollectionMetadataWriter(metadata);
 }

        public EdgeParser(ILexer lexer)
        {
            this.lexer = lexer;

            assemblies = new HashSet<string>();
            namespaces = new HashSet<string>();
            objects = new Dictionary<string, ObjectNode>();
        }

 static void RunQuery(string query, ILexer lexer)
 {
     Console.WriteLine(query);
     foreach (var token in lexer.Tokenize(query))
     {
         Console.WriteLine(token);
     }
 }

        public void SetupLexer(ILexer lexer)
        {
            if (lexer == null)
                throw new ArgumentNullException("lexer");

            lexer.IgnoreWhitespace = true;
            lexer.TokenDefinitions = _tokenDefinitions;
        }