C# Lexer类代码示例

        internal override void Detect(Lexer l)
        {
            if (l.Text.TrimStart().StartsWith("#") &&
                (l.MatchNext(" ") || l.MatchNext("\n")) &&
                l.Text != "#import" &&
                l.Text != "#include")
                l.TakeOwnership();
            else if (!l.Text.TrimStart().StartsWith("#") || l.Text == "#import" || l.Text == "#include")
                l.ForceExclude();

            if (l.HasOwnership())
            {
                if (l.Char == '\\' && l.MatchNext("\n"))
                {
                    this.m_KeepOwnership = true;
                }
                else if (l.Char == '\n' && !this.m_KeepOwnership)
                {
                    l.AddNode(new DirectNode(l.Text));
                    l.EndOwnership();
                }
                else if (this.m_KeepOwnership)
                    this.m_KeepOwnership = false;
            }
        }

        private static Nodes.Assignment ArrayAssignment(Lexer.Lexem currLexem, Nodes.Node.Coords coords)
        {
            Nodes.Expression index = Expr();
            if (Lexer.LookAhead().LexType != Lexer.LexType.CloseSquadBracket)
                ErrorList.Add(new Error(ParserException.NeedCloseSquadBracket, Lexer.LookBack().EndCoords));
            else
                Lexer.NextLexem();

            if (Lexer.LookAhead().LexType == Lexer.LexType.EqualSign)
            {
                Lexer.NextLexem();
                Nodes.Expression value = Expr();
                /*if (Lexer.LookAhead().LexType != Lexer.LexType.Semicolon)
                    ErrorList.Add(new Error(ParserException.NeedSemicolon, Lexer.LookBack().EndCoords));
                else
                    Lexer.NextLexem();*/
                Nodes.Assignment result = new Nodes.Assignment(new Nodes.ElementOfArray(currLexem.VarName, index), value, coords);
                result.Head = result;
                result.Tail = result;
                return result;
            }
            else
            {
                while (Lexer.LookBack().LexType != Lexer.LexType.Variable)
                    Lexer.RollBack();
                Lexer.RollBack();
                return null;
            }
        }

 public void Lexer_CanIdentifyNumberTokens(string input, double expectedValue)
 {
     var lexer = new Lexer(input);
     Assert.IsTrue(lexer.Next());
     Assert.AreEqual(TokenType.Number, lexer.CurrentToken.Type);
     Assert.AreEqual(expectedValue, (double)(Number)lexer.CurrentToken.Value);
 }

    /// <summary>
    /// Count the number of words in this list that matches in the phrase
    /// </summary>
    /// <param name="lexer">The words to match</param>
    /// <param name="count">The number of words that match [out]</param>
    /// <returns>Score for the match: 0 if no match</returns>
    internal double matchMembers(Lexer lexer, out int count)
    {
        var cnt = 0  ; // The number of words that match
          var score=0.0; // The score to match

          // for each of the remaining words
          while (!lexer.EOF)
          {
         lexer.Preprocess();
         var word =lexer.Symbol();

         // See if it matches a modifier
         double score1 = matchMembers(word);

         // check for a minimum score
         if (score1 <= 0.0) break;

         // Update the tracking info
         cnt++;
         score += score1;
          }

          // return the results
          count = cnt;
          return score;
    }

 public static void Main(string[] args)
 {
     while (true)
     {
         try
         {
             string s = Console.ReadLine();
             Lexer l = new Lexer();
             Parser p = new Parser(l.Lex(s));
             Ast.Chunk c = p.Parse();
             Compiler.Compiler cplr = new SharpLua.Compiler.Compiler();
             LuaFile proto = cplr.Compile(c, "<stdin>");
             Console.WriteLine("compiled!");
             FileStream fs = File.Open("out.sluac", FileMode.Create);
             foreach (char ch in proto.Compile())
             {
                 //Console.WriteLine(ch + " " + (int)ch);
                 fs.WriteByte((byte)ch);
             }
             fs.Close();
             Console.WriteLine("written to out.sluac!");
         }
         catch (Exception ex)
         {
             Console.WriteLine(ex.ToString());
         }
     }
     Console.Write("Press any key to continue . . . ");
     Console.ReadKey(true);
 }

 /// <summary>
 /// Finds the object that best matches the phrase
 /// </summary>
 /// <param name="phrase">The words to match</param>
 /// <param name="index">The first word in phrase to match</param>
 /// <param name="score">The score for the best match: 0 if no match[out]</param>
 /// <param name="numWordsMatch">The number of words that match [out]</param>
 /// <param name="bestMatch">The object that best matches [out]</param>
 /// <param name="seen">Set of objects already examined</param>
 public void match(Lexer lexer, ref double score,
            ref int numWordsMatch,
            ref object bestMatch,
            ref LexState lexerState, Dictionary<object,object> seen)
 {
     // Skip if we've already seen this object
       if (seen.ContainsKey(this))
      return;
       // Add ourself to prevent cycles
       seen[this] = this;
     //      edge.match(lexer, ref score, ref numWordsMatch, ref bestMatch, ref lexerState, seen);
     #if true
       /// We only check to see if the gate matches the phrase
       if (null != edge.gate)
      edge.gate.match(lexer, ref score, ref numWordsMatch,
                          ref bestMatch, ref lexerState, seen);
       // Should only match if no door, or door is open
       if (null != edge.sink && (null == edge.gate || isOpen(edge.gate)))
       {
      // we only want match -- matchInContext checks the stuff from the parents..
      edge.sink.match(lexer, ref score, ref numWordsMatch,
                  ref bestMatch, ref lexerState, seen);
       }
     #endif
 }

 public ILexerConfig this[Lexer lexer]
 {
     get
     {
         return this[(int)lexer];
     }
 }

 public void ShouldDistinguishTwoDigitOperatorsStartingWithSameChar()
 {
     var lexer = new Lexer("= ==");
     AssertTokenIs("=", lexer.Next());
     AssertTokenIs("==", lexer.Next());
     AssertTokenIs("EOF", lexer.Next());
 }

 public void ShouldRecognizeNumbers()
 {
     var lexer = new Lexer("1 2.3 4.56789");
     AssertTokenIs("number", 1, lexer.Next());
     AssertTokenIs("number", 2.3, lexer.Next());
     AssertTokenIs("number", 4.56789, lexer.Next());
 }

        public override ParseTree Parse(Lexer lexer, ParserState state)
        {
            state.RuntimeState.Runtime.ParseTrace.Enter(this, lexer.CurrentSource(), "Pattern " + Type.Name);
            
            int start = lexer.Position;
            
            if (state.Excluded.Contains(this))
            {
                state.RuntimeState.Runtime.ParseTrace.No(this, lexer.CurrentSource(), "Excluded");
                return ParseTree.No;
            }
            
		    Precedence oldCurrentPrecedence = state.CurrentPrecedence;
		    
		    if (Precedence.Overwrites(state.CurrentPrecedence))
		        state.CurrentPrecedence = Precedence;

            ParseTree tree = ParseGraph.Parse(lexer, state);
            
            state.CurrentPrecedence = oldCurrentPrecedence;

            if (tree == ParseTree.No)
            {
                state.RuntimeState.Runtime.ParseTrace.No(this, lexer.SourceFrom(start));
                return ParseTree.No;
            }

            state.RuntimeState.Runtime.ParseTrace.Yes(this, lexer.SourceFrom(start));
            
            return tree;
        }

        protected override ResultOfProcess ProcessToken(Lexer.Token token)
        {
            this.Context.Recovery(token);

            var tmpExpressionStatementLeaf = new Syntaxer.StatementLeafs.ExpressionStatementLeaf(this.Context);

            var tmpRez = tmpExpressionStatementLeaf.Run();

            var tmpResultNode = tmpExpressionStatementLeaf.Result;

            if (mCurrStatement == null)
            {
                mCurrStatement = tmpResultNode;

                mMainCodeBlock.FirstStatement = mCurrStatement;
            }
            else
            {
                mCurrStatement.NextStatement = tmpResultNode;

                mCurrStatement = tmpResultNode;
            }

            return tmpRez;
        }

        public void CorrectLineAndColumnNumbers()
        {
            var input = "Hello World!\nSecond line\r\nFoo";

            MemoryStream m = new MemoryStream();
            var w = new StreamWriter(m);
            w.Write(input);
            w.Flush();
            m.Position = 0;

            var word = new Terminal("Word", "\\S+");
            var whitespace = new Terminal("Whitespace", " |\n|\r");

            var lexer = new Lexer(m, word, whitespace);
            Token[] tokens = lexer.ToArray();

            Assert.AreEqual(10, tokens.Length);
            AssertToken(tokens[0], "Word", 1, 1);
            AssertToken(tokens[1], "Whitespace", 1, 6);
            AssertToken(tokens[2], "Word", 1, 7);
            AssertToken(tokens[3], "Whitespace", 1, 13);
            AssertToken(tokens[4], "Word", 2, 1);
            AssertToken(tokens[5], "Whitespace", 2, 7);
            AssertToken(tokens[6], "Word", 2, 8);
            AssertToken(tokens[7], "Whitespace", 2, 12);
            AssertToken(tokens[8], "Whitespace", 2, 13);
            AssertToken(tokens[9], "Word", 3, 1);
        }

 public override ParseTree Parse(Lexer lexer, ParserState state)
 {
     lexer.Whitespace(state.RuntimeState);
     
     int start = lexer.Position;
     
     state.RuntimeState.Runtime.ParseTrace.Enter(this, lexer.CurrentSource(), range.ToString());
     
     char character = lexer.Peek();
     
     if (!range.Contains(character))
     {
         lexer.ErrorStrings[start].Add(range.ToString());
         
         state.RuntimeState.Runtime.ParseTrace.No(this,
             lexer.SourceFrom(start), TextEscape.Quote(character));
         
         return ParseTree.No;
     }
     
     lexer.Read();
     
     state.RuntimeState.Runtime.ParseTrace.Yes(this,
         lexer.SourceFrom(start), TextEscape.Quote(character));
     
     if (state.BuildTextNodes)
         return new ParseTree(Convert.ToString(character));
     else
         return ParseTree.Yes;
 }

        /// <summary>
        /// Parses the given chunk of code and verifies that it matches the expected pretty-printed result
        /// </summary>
        /// <param name="source"></param>
        /// <param name="expected"></param>
        public void Test(string source, string expected)
        {
            var lexer = new Lexer(source);
            Parser parser = new BantamParser(lexer);

            try
            {
                var result = parser.ParseExpression();
                var builder = new StringBuilder();
                result.Print(builder);
                var actual = builder.ToString();

                if (expected.Equals(actual))
                {
                    _passed++;
                }
                else
                {
                    _failed++;
                    Console.Out.WriteLine("[FAIL] Expected: " + expected);
                    Console.Out.WriteLine("         Actual: " + actual);
                }
            }
            catch (ParseException ex)
            {
                _failed++;
                Console.Out.WriteLine("[FAIL] Expected: " + expected);
                Console.Out.WriteLine("          Error: " + ex.Message);
            }
        }

 public ModuleDefinitionLoader(IServiceProvider services, string filename, byte[] bytes) : base(services, filename, bytes)
 {
     this.filename = filename;
     this.lexer = new Lexer( new StreamReader(new MemoryStream(bytes)));
     this.bufferedTok = null;
     this.arch = new Reko.Arch.X86.X86ArchitectureFlat32();
 }