C# Scanner.SetSource方法代码示例

        public override Microsoft.VisualStudio.Package.AuthoringScope ParseSource(ParseRequest req)
        {
            Babel.Source source = (Babel.Source)this.GetSource(req.FileName);
            bool yyparseResult = false;

            // req.DirtySpan seems to be set even though no changes have occurred
            // source.IsDirty also behaves strangely
            // might be possible to use source.ChangeCount to sync instead

            if (req.DirtySpan.iStartIndex != req.DirtySpan.iEndIndex
                || req.DirtySpan.iStartLine != req.DirtySpan.iEndLine) {
                ErrorHandler handler = new ErrorHandler();
                Scanner scanner = new Scanner(); // string interface
                Parser parser = new Parser();  // use noarg constructor
                parser.scanner = scanner;
                scanner.Handler = handler;
                parser.SetHandler(handler);
                scanner.SetSource(req.Text, 0);

                parser.MBWInit(req);
                yyparseResult = parser.Parse();

                // store the parse results
                // source.ParseResult = aast;
                source.ParseResult = null;
                source.Braces = parser.Braces;

                // for the time being, just pull errors back from the error handler
                if (handler.ErrNum > 0) {
                    foreach (Babel.Parser.Error error in handler.SortedErrorList()) {
                        TextSpan span = new TextSpan();
                        span.iStartLine = span.iEndLine = error.line - 1;
                        span.iStartIndex = error.column;
                        span.iEndIndex = error.column + error.length;
                        req.Sink.AddError(req.FileName, error.message, span, Severity.Error);
                    }
                }
            }
            switch (req.Reason) {
                case ParseReason.Check:
                case ParseReason.HighlightBraces:
                case ParseReason.MatchBraces:
                case ParseReason.MemberSelectAndHighlightBraces:
                    // send matches to sink
                    // this should (probably?) be filtered on req.Line / col
                    if (source.Braces != null) {
                        foreach (TextSpan[] brace in source.Braces) {
                            if (brace.Length == 2)
                                req.Sink.MatchPair(brace[0], brace[1], 1);
                            else if (brace.Length >= 3)
                                req.Sink.MatchTriple(brace[0], brace[1], brace[2], 1);
                        }
                    }
                    break;
                default:
                    break;
            }

            return new AuthoringScope(req.Text);
        }

        protected static Parser GetNewParser(ScannerOptions scannerOptions, string template)
        {
            Scanner scanner = new Scanner(new ErrorHandler());
            scanner.Options = scannerOptions;
            scanner.SetSource(template);

            return new Parser(scanner, new ErrorHandler());
        }

        public ForkableScanner Create(SourceReader source)
        {
            CodeContract.RequiresArgumentNotNull(source, "source");

            Scanner masterScanner = new Scanner(ScannerInfo);
            masterScanner.SetSource(source);
            masterScanner.SetTriviaTokens(m_triviaTokens);
            masterScanner.ErrorList = ErrorList;
            masterScanner.RecoverErrors = RecoverErrors;
            masterScanner.LexicalErrorId = LexicalErrorId;

            return ForkableScanner.Create(masterScanner);
        }

        /// <summary>
        /// Generates Transact-SQL batches, suitable for execution, from the text of a 
        /// Transact-SQL script.
        /// </summary>
        /// <param name="scriptSql">Transact-SQL script text</param>
        /// <returns>Collection of Batch objects for each batch in the supplied script</returns>
        public static IEnumerable<Batch> GetBatches(string scriptSql)
        {
            /* This method could be rewritten to use the new `Parser.ParseSql` class method. */

            ParseOptions parseOptions = new ParseOptions();
            Scanner scanner = new Scanner(parseOptions);

            int state = 0,
                start,
                end,
                lastTokenEnd = -1,
                token;

            bool isPairMatch, isExecAutoParamHelp;

            List<Batch> batches = new List<Batch>();
            StringBuilder nextBatchSql = new StringBuilder();

            scanner.SetSource(scriptSql, 0);

            while ((token = scanner.GetNext(ref state, out start, out end, out isPairMatch, out isExecAutoParamHelp)) != (int)Tokens.EOF)
            {
                if ((Tokens)token == Tokens.LEX_BATCH_SEPERATOR)
                {
                    nextBatchSql.Append(scriptSql.Substring(lastTokenEnd + 1, start - lastTokenEnd + 1 - 1 - 1));
                    batches.Add(new Batch(nextBatchSql.ToString()));
            #if NET35
                    nextBatchSql = new StringBuilder();
            #else
                    nextBatchSql.Clear();
            #endif
                }
                else
                {
                    nextBatchSql.Append(scriptSql.Substring(lastTokenEnd + 1, end - lastTokenEnd + 1 - 1));
                }

                lastTokenEnd = end;
            }

            #if NET35
            if (!String.IsNullOrEmpty(nextBatchSql.ToString().Trim()))
            #else
            if (!String.IsNullOrWhiteSpace(nextBatchSql.ToString()))
            #endif
                batches.Add(new Batch(nextBatchSql.ToString()));

            return batches;
        }

    public static object StringToNumber(object obj)
    {
      string str = RequiresNotNull<string>(obj);

      if (str.Length == 0)
      {
        return FALSE;
      }

      Parser number_parser = new Parser();
      Scanner number_scanner = new Scanner();

      number_parser.scanner = number_scanner;
      number_scanner.SetSource(str, 0);
      number_parser.result = null;
      number_scanner.yy_push_state(3);

      try
      {
        CallTarget0 disp = delegate
        {
          if (number_parser.Parse())
          {
            Debug.Assert(number_parser.result != null);
            return number_parser.result;
          }
          else
          {
            return FALSE;
          }
        };
        CallTarget1 handler = delegate(object o)
        {
          throw new Continuation();
        };
        return Runtime.R6RS.Exceptions.WithExceptionHandler(
                          Closure.Create(handler),
                          Closure.Create(disp));
      }
      catch
      {
        return FALSE;
      }
    }

        public static IEnumerable<TokenInfo> ParseSql(string sql)
        {
            ParseOptions parseOptions = new ParseOptions();
            Scanner scanner = new Scanner(parseOptions);

            int state = 0,
                start,
                end,
                lastTokenEnd = -1,
                token;

            bool isPairMatch, isExecAutoParamHelp;

            List<TokenInfo> tokens = new List<TokenInfo>();
            StringBuilder nextBatchSql = new StringBuilder();

            scanner.SetSource(sql, 0);

            while ((token = scanner.GetNext(ref state, out start, out end, out isPairMatch, out isExecAutoParamHelp)) != (int)Tokens.EOF)
            {
                TokenInfo tokenInfo =
                    new TokenInfo()
                    {
                        Start = start,
                        End = end,
                        IsPairMatch = isPairMatch,
                        IsExecAutoParamHelp = isExecAutoParamHelp,
                        Sql = sql.Substring(start, end - start + 1),
                        Token = (Tokens)token,
                    };

                tokens.Add(tokenInfo);

                lastTokenEnd = end;
            }

            return tokens;
        }

 private void ParseItems()
 {
     Scanner scanner = new Scanner();
     scanner.SetSource(source, 0);
     int state=0,start,end;
     int tokenType = -1;
     while (tokenType != ((int)Tokens.EOF)) {
         tokenType = scanner.GetNext(ref state, out start, out end);
         if (tokenType == (int)Tokens.LCASEIDENT || tokenType == (int)Tokens.OUTACTION) {
             string channel = scanner.yytext.Replace("_", "");
             if (!channels.Contains(channel)) {
                 channels.Add(channel);
             }
         } else if (tokenType == (int)Tokens.PROC) {
             if (!processes.Contains(scanner.yytext)) {
                 processes.Add(scanner.yytext);
             }
         } else if (tokenType == (int)Tokens.FULLCLASS) {
             if (!importedClasses.Contains(scanner.yytext)) {
                 importedClasses.Add(scanner.yytext);
             }
         }
     }
 }

        public static void Main(string[] args)
        {
            Scanner scanner = new Scanner(new ErrorHandler());
            string fileName;

            if (args.Length == 1 && args[0] == "/parser-gui")
            {
                Console.WriteLine("Loading Parser GUI");
                Application.EnableVisualStyles();
                Application.SetCompatibleTextRenderingDefault(false);
                Application.Run(new ParserGUIForm());
                return;
            }

            Console.WindowHeight = 50;
            Console.WriteLine("Castle.NVelocity Demo");
            Console.WriteLine("=====================");

            if (args.Length > 0)
            {
                fileName = args[0];
            }
            else
            {
                Console.WriteLine("No file specified. Enter file name:");
                fileName = Console.ReadLine();
            }

            if (string.IsNullOrEmpty(fileName))
            {
                Console.WriteLine("Filename is null, exiting...");
                return;
            }

            Console.WriteLine("Loading: {0}", fileName);
            string source = File.ReadAllText(fileName);

            Console.WriteLine();
            scanner.SetSource(source);

            try
            {
                while (!scanner.EOF)
                {
                    Token token = scanner.GetToken();
                    if (token != null)
                    {
                        switch (token.Type)
                        {
                            case TokenType.XmlText:
                                Console.ForegroundColor = ConsoleColor.Yellow;
                                break;
                            case TokenType.XmlComment:
                                Console.ForegroundColor = ConsoleColor.Green;
                                break;
                            case TokenType.XmlTagName:
                                Console.ForegroundColor = ConsoleColor.Cyan;
                                break;
                            case TokenType.XmlTagStart:
                            case TokenType.XmlTagEnd:
                            case TokenType.XmlForwardSlash:
                            case TokenType.XmlEquals:
                            case TokenType.XmlDoubleQuote:
                            case TokenType.XmlQuestionMark:
                            case TokenType.XmlExclaimationMark:
                                Console.ForegroundColor = ConsoleColor.Red;
                                break;

                            case TokenType.NVDirectiveHash:
                            case TokenType.NVDirectiveName:
                            case TokenType.NVLParen:
                            case TokenType.NVRParen:
                            case TokenType.NVLBrack:
                            case TokenType.NVRBrack:
                            case TokenType.NVLCurly:
                            case TokenType.NVRCurly:
                            case TokenType.NVDictionaryPercent:
                                Console.ForegroundColor = ConsoleColor.Magenta;
                                break;
                            case TokenType.NVTrue:
                            case TokenType.NVFalse:
                            case TokenType.NVIn:
                            case TokenType.NVWith:
                                Console.ForegroundColor = ConsoleColor.Blue;
                                break;
                            case TokenType.NVDollar:
                            case TokenType.NVIdentifier:
                            case TokenType.NVDictionaryKey:
                            case TokenType.NVReferenceLCurly:
                            case TokenType.NVReferenceRCurly:
                            case TokenType.NVReferenceSilent:
                                Console.ForegroundColor = ConsoleColor.Cyan;
                                break;
                            case TokenType.NVSingleLineComment:
                                Console.ForegroundColor = ConsoleColor.Green;
                                break;
                            case TokenType.NVDoubleQuote:
                            case TokenType.NVSingleQuote:
                                Console.ForegroundColor = ConsoleColor.Red;
                                break;
//.........这里部分代码省略.........

        public void IncompleteXmlTagWithNoNameInsideXmlTagBreaksOutOfXmlTagWhenInIntelliSenseMode()
        {
            Scanner scanner = new Scanner(new ErrorHandler());
            ScannerOptions scannerOptions = new ScannerOptions();
            scannerOptions.EnableIntelliSenseTriggerTokens = true;
            scanner.Options = scannerOptions;

            scanner.SetSource(
                "<div>\n" +
                "    < \n" +
                "</div>");

            AssertMatchToken(scanner, TokenType.XmlTagStart);
            AssertMatchToken(scanner, TokenType.XmlTagName, "div");
            AssertMatchToken(scanner, TokenType.XmlTagEnd);

            AssertMatchToken(scanner, TokenType.XmlText, "\n    ");
            AssertMatchToken(scanner, TokenType.XmlTagStart);
            AssertMatchToken(scanner, TokenType.XmlAttributeMemberSelect, " ");
            AssertMatchToken(scanner, TokenType.XmlAttributeMemberSelect, "\n");
            // When in IntelliSense mode the scanner should drop out of a tag if it finds a '<'

            AssertMatchToken(scanner, TokenType.XmlTagStart);
            AssertMatchToken(scanner, TokenType.XmlForwardSlash);
            AssertMatchToken(scanner, TokenType.XmlTagName, "div");
            AssertMatchToken(scanner, TokenType.XmlTagEnd);

            AssertEOF();
        }

        public override int ValidateBreakpointLocation(IVsTextBuffer buffer, int line, int col, TextSpan[] pCodeSpan)
        {
            if (pCodeSpan != null) {
                pCodeSpan[0].iStartLine = line;
                pCodeSpan[0].iEndLine = line;
                List<TokenInfo> tokens = new List<TokenInfo>();
                if (buffer != null) {
                    int length;
                    buffer.GetLengthOfLine(line, out length);
                    IVsTextLines lines = (IVsTextLines)buffer;
                    string text;
                    lines.GetLineText(line, 0, line, length, out text);

                    Scanner scanner = new Scanner();
                    scanner.SetSource(text, 0);
                    int state = 0, start, end;
                    int tokenType = -1;
                    while (tokenType != ((int)Tokens.EOF)) {
                        tokenType = scanner.GetNext(ref state, out start, out end);
                        tokens.Add(new TokenInfo(start, end, tokenType, scanner.yytext));
                    }

                    for (int i = 0; i < tokens.Count; i++) {
                        TokenInfo t = tokens[i];
                        if (t.type == (int)Tokens.OUTACTION
                            || t.type == (int)Tokens.METHOD
                            || t.type == (int)Tokens.LCASEIDENT
                            || (t.type == (int)Tokens.NUMBER && t.text == "0")) {
                            pCodeSpan[0].iStartIndex = t.start;
                            pCodeSpan[0].iEndIndex = t.end + 1;
                            return Microsoft.VisualStudio.VSConstants.S_OK;
                        } else if (t.type == (int)Tokens.PROC) {
                            //Just a heuristic, if there is a '=' somewhere after this, then we don't want it
                            //Most likely it is Foo = proc or Foo(x,y,z) = proc and we don't want to highlight that
                            bool canUseProc = true;
                            for (int j = i + 1; j < tokens.Count; j++) {
                                if (tokens[j].type == (int)'=') {
                                    i = j;
                                    canUseProc = false;
                                    break;
                                }
                            }
                            if (canUseProc) {
                                pCodeSpan[0].iStartIndex = t.start;
                                pCodeSpan[0].iEndIndex = t.end + 1;
                                return Microsoft.VisualStudio.VSConstants.S_OK;
                            }
                        }

                    }
                }
            }
            return Microsoft.VisualStudio.VSConstants.S_FALSE;
        }

        static void Main(string[] args)
        {
            try
            {
                var scanner = new Scanner();
                var parser = new Parser(scanner);

                scanner.SetSource(File.OpenRead("test.lae"));
                if ( parser.Parse() )
                {
                    String files = String.Empty;
                    String init = String.Empty;
                    String s = String.Empty;

                    foreach ( var item in parser.ParseTree )
                    {
                        String stateName = item.Target.Name;
                        String startCondition = item.Expression.Condition.Compile();
                        String beforeRun = String.Empty;
                        String afterRun = String.Empty;

                        bool before = true;

                        for ( var i = 0; i < item.Expression.Iteration.Actors.Count; i++ )
                        {
                            if ( item.Expression.Iteration.Actors[i].GetType() == typeof( ExprCall ) )
                            {
                                before = false;
                                continue;
                            }

                            if ( before )
                            {
                                beforeRun += item.Expression.Iteration.Actors[i].Compile() + ";\n";
                            } else {
                                afterRun += item.Expression.Iteration.Actors[i].Compile() + ";\n";
                            }
                        }

                        s = File.ReadAllText( "..\\..\\Templates\\CommonState.cs" );

                        s = s.Replace( "##StateName##", stateName );
                        s = s.Replace( "##StartCondition##", startCondition );
                        s = s.Replace( "##BeforeRun##", beforeRun );
                        s = s.Replace( "##AfterRun##", afterRun );

                        Directory.CreateDirectory( "Solution" );
                        File.WriteAllText( "Solution\\" + stateName + ".cs", s );

                        files += "<Compile Include=\"" + stateName + ".cs\" />\n";
                        init += "var s" + number.ToString() + " = new " + stateName + "();\n";
                        number++;
                    }

                    s = File.ReadAllText( "..\\..\\Templates\\Solution.csproj" );
                    s = s.Replace( "##Files##", files );
                    File.WriteAllText( "Solution\\Solution.csproj", s );

                    s = File.ReadAllText( "..\\..\\Templates\\Startup.cs" );
                    s = s.Replace( "##Init##", init );
                    File.WriteAllText( "Solution\\Startup.cs", s );

                    File.Copy( "..\\..\\Templates\\State.cs", "Solution\\State.cs", true );
                    File.Copy( "..\\..\\Templates\\Solution.sln", "Solution\\Solution.sln", true );
                }

            }
            catch (Exception ex)
            {
                Console.WriteLine(ex);
            }

            //Console.WriteLine("Press ENTER ...");
            //Console.ReadLine();
        }

        public static string ReplaceNonBreakingSpaces(string scriptSql)
        {
            /* This method could be rewritten to use the new `Parser.ParseSql` class method. */

            ParseOptions parseOptions = new ParseOptions();
            Scanner scanner = new Scanner(parseOptions);

            int state = 0,
                start,
                end,
                lastTokenEnd = -1,
                token;

            bool isPairMatch, isExecAutoParamHelp;

            StringBuilder newScriptSql = new StringBuilder();

            scanner.SetSource(scriptSql, 0);

            while ((token = scanner.GetNext(ref state, out start, out end, out isPairMatch, out isExecAutoParamHelp)) != (int)Tokens.EOF)
            {
                newScriptSql.Append(scriptSql.Substring(lastTokenEnd + 1, start - lastTokenEnd - 1));

                string tokenSubstring = scriptSql.Substring(start, end - start + 1);

                if ((Tokens)token != Tokens.TOKEN_STRING)
                {
                    newScriptSql.Append(tokenSubstring.Replace('\xA0', '\x20'));
                }
                else
                {
                    newScriptSql.Append(tokenSubstring);
                }

                lastTokenEnd = end;
            }

            if (lastTokenEnd + 1 < scriptSql.Length)
                newScriptSql.Append(scriptSql.Substring(lastTokenEnd + 1));

            return newScriptSql.ToString();
        }