C# Scanner.GetToken方法代码示例

 public void TestScanner(string dataFile, string canonFile)
 {
     var s = new Scanner(new Reader(new FileStream(dataFile, FileMode.Open)));
     var tokens1 = new List<Tokens.Base>();
     while(s.CheckToken())
         tokens1.Add(s.GetToken());
     s = new Scanner(new Reader(new FileStream(canonFile, FileMode.Open)));
     var tokens2 = new List<Tokens.Base>();
     while (s.CheckToken())
         tokens2.Add(s.GetToken());
 }

        public void ParseTest1()
        {
            string exp = "1 + 2 * (3 - 4 - 5)";
            Scanner sc = new Scanner(exp);
            List<Token> tks = new List<Token>();
            Token tk = sc.GetToken();
            while (tk != null)
            {
                tks.Add(tk);
                tk = sc.GetToken();
            }

            string res = "";
            foreach (Token t in tks)
            {
                res += t.ToString();
            }

            string expected = "1+2*(3-4-5)";
            Assert.AreEqual(expected, res);
        }

 public void TestScanner(string dataFile, string canonFile)
 {
     var file = new FileStream(dataFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
     var s = new Scanner(new Reader(file));
     var tokens1 = new List<Tokens.Base>();
     while(s.CheckToken())
         tokens1.Add(s.GetToken());
     file.Close();
     file = new FileStream(canonFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
     s = new Scanner(new Reader(file));
     var tokens2 = new List<Tokens.Base>();
     while (s.CheckToken())
         tokens2.Add(s.GetToken());
     file.Close();
 }

        /// <summary>
        /// Parse identifier list. Identifier list have the syntax: identifierList ::= stringDelimiter identifier* stringDelimiter
        /// </summary>
        /// <param name="token">Token containing the string with the identifier list.</param>
        /// <param name="parseErrorSink">Error sink for reporting parse errors.</param>
        /// <param name="sourceCodeService">Source code service that can convert tokens to source code span and reports issues.</param>
        /// <returns>A collection of sub-tokens if successful or null if token string is illegal.</returns>
        protected internal virtual IEnumerable<SourceReference<string>> ParseIdentifierList(StringToken token, IParseErrorSink parseErrorSink, ISourceCodeReferenceService sourceCodeService)
        {
            if (token == null)
                throw new ArgumentNullException("token");
            if (parseErrorSink == null)
                throw new ArgumentNullException("parseErrorSink");
            if (sourceCodeService == null)
                throw new ArgumentNullException("sourceCodeService");

            List<SourceReference<string>> result = new List<SourceReference<string>>();
            if (token.Value.Length == 0)
                return result;

            Scanner scanner = new Scanner(new StringReader(token.Value));
            Token t = scanner.GetToken();
            while (!(t is EofToken))
            {
                // Skip whitespaces (incl. comments).
                if (!(t is WhitespaceToken))
                {
                    // Offset the token to the position within the original token. +1 is due to the opening string ' quote.
                    t.SetTokenValues(
                        new SourceLocation(t.StartPosition.Position + token.StartPosition.Position + 1,
                            t.StartPosition.Line + token.StartPosition.Line,
                            t.StartPosition.Column + token.StartPosition.Column + 1),
                        new SourceLocation(t.StopPosition.Position + token.StartPosition.Position + 1,
                            t.StopPosition.Line + token.StartPosition.Line,
                            t.StopPosition.Column + token.StartPosition.Column + 1),
                        t.ScanError);

                    if (t is IdentifierToken)
                    {
                        // identifier  OK
                        result.Add(new SourceReference<string>(((IdentifierToken)t).Value, t.StartPosition, t.StopPosition, sourceCodeService));
                    }
                    else
                    {
                        // Non-identifier, report error
                        parseErrorSink.AddParserError(t.StartPosition, t.StopPosition, "Expected identifier", t);
                        return null; // Error condition
                    }
                }
                t = scanner.GetToken();
            }
            return result;
        }

        public void ParseTest2()
        {
            string exp = "1 + 2 * $s(#12, #24) + (@3 - @4 - 5) + $c(@11 >= 15, 100, 200)";
            Scanner sc = new Scanner(exp);
            List<Token> tks = new List<Token>();
            Token tk = sc.GetToken();
            while (tk != null)
            {
                tks.Add(tk);
                tk = sc.GetToken();
            }

            string res = "";
            foreach (Token t in tks)
            {
                res += t.ToString();
            }

            string expected = "1+2*$s(#12,#24)+(@[email protected])+$c(@11>=15,100,200)";
            Assert.AreEqual(expected, res);
        }

        private void btnTokanize_Click(object sender, EventArgs e)
        {
            this.listTokens.Items.Clear();

            string txt = this.txtSource.SelectedText;
            if (String.IsNullOrEmpty(txt))
                txt = this.txtSource.Text;
            StringReader reader = new StringReader(txt);
            Scanner lexer = new Scanner(reader);

            Preference preference = Preference.Default;
            if (this.comboPreference.SelectedIndex == 1)
                preference = Preference.NegativeSign;
            if (this.comboPreference.SelectedIndex == 2)
                preference = Preference.VerticalBar;
            if (this.comboPreference.SelectedIndex == 3)
                preference = Preference.NegativeSign | Preference.VerticalBar;
            if (this.comboPreference.SelectedIndex == 4)
                preference = Preference.UnquotedSelector;

            Token token;
            while ((token = lexer.GetToken(preference)) != null)
            {
                ListViewItem lvi = new ListViewItem();
                lvi.Text = token.GetType().Name;
                lvi.SubItems.Add(token.StartPosition.ToString());
                lvi.SubItems.Add(token.StopPosition.ToString());
                lvi.SubItems.Add(this.GetTokenValue(token));
                lvi.SubItems.Add(token.SourceString);
                lvi.SubItems.Add(token.IsValid ? "Y" : "N");
                lvi.SubItems.Add(token.ScanError);
                lvi.Tag = token;

                this.listTokens.Items.Add(lvi);
            }
        }

        public static void Main(string[] args)
        {
            Scanner scanner = new Scanner(new ErrorHandler());
            string fileName;

            if (args.Length == 1 && args[0] == "/parser-gui")
            {
                Console.WriteLine("Loading Parser GUI");
                Application.EnableVisualStyles();
                Application.SetCompatibleTextRenderingDefault(false);
                Application.Run(new ParserGUIForm());
                return;
            }

            Console.WindowHeight = 50;
            Console.WriteLine("Castle.NVelocity Demo");
            Console.WriteLine("=====================");

            if (args.Length > 0)
            {
                fileName = args[0];
            }
            else
            {
                Console.WriteLine("No file specified. Enter file name:");
                fileName = Console.ReadLine();
            }

            if (string.IsNullOrEmpty(fileName))
            {
                Console.WriteLine("Filename is null, exiting...");
                return;
            }

            Console.WriteLine("Loading: {0}", fileName);
            string source = File.ReadAllText(fileName);

            Console.WriteLine();
            scanner.SetSource(source);

            try
            {
                while (!scanner.EOF)
                {
                    Token token = scanner.GetToken();
                    if (token != null)
                    {
                        switch (token.Type)
                        {
                            case TokenType.XmlText:
                                Console.ForegroundColor = ConsoleColor.Yellow;
                                break;
                            case TokenType.XmlComment:
                                Console.ForegroundColor = ConsoleColor.Green;
                                break;
                            case TokenType.XmlTagName:
                                Console.ForegroundColor = ConsoleColor.Cyan;
                                break;
                            case TokenType.XmlTagStart:
                            case TokenType.XmlTagEnd:
                            case TokenType.XmlForwardSlash:
                            case TokenType.XmlEquals:
                            case TokenType.XmlDoubleQuote:
                            case TokenType.XmlQuestionMark:
                            case TokenType.XmlExclaimationMark:
                                Console.ForegroundColor = ConsoleColor.Red;
                                break;

                            case TokenType.NVDirectiveHash:
                            case TokenType.NVDirectiveName:
                            case TokenType.NVLParen:
                            case TokenType.NVRParen:
                            case TokenType.NVLBrack:
                            case TokenType.NVRBrack:
                            case TokenType.NVLCurly:
                            case TokenType.NVRCurly:
                            case TokenType.NVDictionaryPercent:
                                Console.ForegroundColor = ConsoleColor.Magenta;
                                break;
                            case TokenType.NVTrue:
                            case TokenType.NVFalse:
                            case TokenType.NVIn:
                            case TokenType.NVWith:
                                Console.ForegroundColor = ConsoleColor.Blue;
                                break;
                            case TokenType.NVDollar:
                            case TokenType.NVIdentifier:
                            case TokenType.NVDictionaryKey:
                            case TokenType.NVReferenceLCurly:
                            case TokenType.NVReferenceRCurly:
                            case TokenType.NVReferenceSilent:
                                Console.ForegroundColor = ConsoleColor.Cyan;
                                break;
                            case TokenType.NVSingleLineComment:
                                Console.ForegroundColor = ConsoleColor.Green;
                                break;
                            case TokenType.NVDoubleQuote:
                            case TokenType.NVSingleQuote:
                                Console.ForegroundColor = ConsoleColor.Red;
                                break;
//.........这里部分代码省略.........

 // A method for testing the other methods in the Scanner class.
 // A test string which contains at least one example of each
 // kind of token is used.
 public static bool UnitTest(bool verbose)
 {
     string testString =
         @"A123 123 1.2345 123.45E-2 .12345E1
         var func if else while return null
         + - * / = == != < <= > >= ( ) { } , . ; // comment \n";
     Token[] expectedTokens = { Token.Ident, Token.DecNum,
             Token.FltNum, Token.FltNum, Token.FltNum,
             Token.KwdVar, Token.KwdFunc, Token.KwdIf, Token.KwdElse,
             Token.KwdWhile, Token.KwdReturn, Token.KwdNull,
             Token.OpPlus, Token.OpMinus, Token.OpStar, Token.OpDivide,
             Token.OpAssign,
             Token.OpEq, Token.OpNe, Token.OpLt,
             Token.OpLe, Token.OpGt, Token.OpGe,
             Token.LPar, Token.RPar, Token.LBrace, Token.RBrace,
             Token.Comma, Token.Period, Token.Semicolon, Token.Eof };
     Scanner s = new Scanner("test-string", testString);
     foreach( Token t in expectedTokens ) {
         Token tt = s.GetToken();
         if (verbose)
             Console.WriteLine("Test: token = {0}, text = '{1}'", tt, s.TokenText);
         if (t != tt) {
             Console.WriteLine("Scanner: unit test failed on token {0}", t);
             return false;
         }
     }
     Console.WriteLine("Scanner: unit test succeeded");
     return true;
 }

 public void ParseTest3()
 {
     string exp = "&1 + 2 * $s(#12, #24) + (@3 - @4 - 5)+$c(@11 >= 15, 100, 200)";
     Scanner sc = new Scanner(exp);
     List<Token> tks = new List<Token>();
     Token tk = sc.GetToken();
     while (tk != null)
     {
         tks.Add(tk);
         tk = sc.GetToken();
     }
 }

 protected static void AssertMatchToken(Scanner scanner, TokenType tokenType, string image)
 {
     Token token = scanner.GetToken();
     CheckTokenType(token, tokenType);
     CheckImage(token, image);
 }

        private List<string> UsedItemId(string exp)
        {
            List<string> ids = new List<string>();

            Scanner scn = new Scanner(exp);
            Token tk = scn.GetToken();
            while (null != tk)
            {
                if (tk.Type.Equals(TokenType.Id))
                {
                    ids.Add(tk.Value.ToString());
                }
                tk = scn.GetToken();
            }

            return ids;
        }

 public void Parse(string exp)
 {
     tkes.Clear();
     Scanner scn = new Scanner(exp);
     Token tk = scn.GetToken();
     while (null != tk)
     {
         tkes.Add(new TokenEx(tk, GetShownString(tk)));
         tk = scn.GetToken();
     }
 }